Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceConst.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceConst.cs	(revision 8477)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceConst.cs	(revision 8477)
@@ -0,0 +1,79 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "CovarianceConst",
+    Description = "Constant covariance function for Gaussian processes.")]
+  public class CovarianceConst : Item, ICovarianceFunction {
+    [Storable]
+    private double sf2;
+    public double Scale { get { return sf2; } }
+
+    [StorableConstructor]
+    protected CovarianceConst(bool deserializing)
+      : base(deserializing) {
+    }
+
+    protected CovarianceConst(CovarianceConst original, Cloner cloner)
+      : base(original, cloner) {
+      this.sf2 = original.sf2;
+    }
+
+    public CovarianceConst()
+      : base() {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceConst(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return 1;
+    }
+
+    public void SetParameter(double[] hyp) {
+      this.sf2 = Math.Exp(2 * hyp[0]);
+    }
+    public void SetData(double[,] x) {
+      // nothing to do
+    }
+
+
+    public void SetData(double[,] x, double[,] xt) {
+      // nothing to do
+    }
+
+    public double GetCovariance(int i, int j) {
+      return sf2;
+    }
+
+    public double GetGradient(int i, int j, int k) {
+      if (k != 0) throw new ArgumentException("CovarianceConst has only one hyperparameters", "k");
+      return 2 * sf2;
+    }
+  }
+}
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceLinear.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceLinear.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceLinear.cs	(revision 8477)
@@ -29,6 +29,4 @@
   [Item(Name = "CovarianceLinear", Description = "Linear covariance function for Gaussian processes.")]
   public class CovarianceLinear : Item, ICovarianceFunction {
-    private static readonly double[] emptyArray = new double[0];
-
     [Storable]
     private double[,] x;
@@ -89,6 +87,6 @@
     }
 
-    public double[] GetGradient(int i, int j) {
-      return emptyArray;
+    public double GetGradient(int i, int j, int k) {
+      throw new NotSupportedException("CovarianceLinear does not have hyperparameters.");
     }
 
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceNoise.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceNoise.cs	(revision 8477)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceNoise.cs	(revision 8477)
@@ -0,0 +1,83 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "CovarianceNoise",
+    Description = "Noise covariance function for Gaussian processes.")]
+  public class CovarianceNoise : Item, ICovarianceFunction {
+    [Storable]
+    private double sf2;
+    public double Scale { get { return sf2; } }
+
+    [StorableConstructor]
+    protected CovarianceNoise(bool deserializing)
+      : base(deserializing) {
+    }
+
+    protected CovarianceNoise(CovarianceNoise original, Cloner cloner)
+      : base(original, cloner) {
+      this.sf2 = original.sf2;
+    }
+
+    public CovarianceNoise()
+      : base() {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceNoise(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return 1;
+    }
+
+    public void SetParameter(double[] hyp) {
+      this.sf2 = Math.Exp(2 * hyp[0]);
+    }
+    public void SetData(double[,] x) {
+      // nothing to do
+    }
+
+
+    public void SetData(double[,] x, double[,] xt) {
+      // nothing to do
+    }
+
+    public double GetCovariance(int i, int j) {
+      if (i == j) return sf2;
+      else return 0.0;
+    }
+
+    public double GetGradient(int i, int j, int k) {
+      if (k != 0) throw new ArgumentException("CovarianceConst has only one hyperparameters", "k");
+      if (i == j)
+        return 2 * sf2;
+      else
+        return 0.0;
+    }
+  }
+}
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovariancePeriodic.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovariancePeriodic.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovariancePeriodic.cs	(revision 8477)
@@ -35,8 +35,11 @@
     [Storable]
     private double sf2;
+    public double Scale { get { return sf2; } }
     [Storable]
     private double l;
+    public double Length { get { return l; } }
     [Storable]
     private double p;
+    public double Period { get { return p; } }
 
     private bool symmetric;
@@ -74,6 +77,5 @@
       this.p = Math.Exp(hyp[1]);
       this.sf2 = Math.Exp(2 * hyp[2]);
-
-      sf2 = Math.Min(10E6, sf2); // upper limit for the scale
+      // sf2 = Math.Min(10E6, sf2); // upper limit for the scale
 
       sd = null;
@@ -102,40 +104,26 @@
     }
 
+    public double GetGradient(int i, int j, int k) {
+      double v = Math.PI * sd[i, j] / p;
+      switch (k) {
+        case 0: {
+            double newK = Math.Sin(v) / l;
+            newK = newK * newK;
+            return 4 * sf2 * Math.Exp(-2 * newK) * newK;
+          }
+        case 1: {
+            double r = Math.Sin(v) / l;
+            return 4 * sf2 / l * Math.Exp(-2 * r * r) * r * Math.Cos(v) * v;
+          }
+        case 2: {
+            double newK = Math.Sin(v) / l;
+            newK = newK * newK;
+            return 2 * sf2 * Math.Exp(-2 * newK);
 
-    public double[] GetDiagonalCovariances() {
-      if (x != xt) throw new InvalidOperationException();
-      int rows = x.GetLength(0);
-      var cov = new double[rows];
-      for (int i = 0; i < rows; i++) {
-        double k = Math.Sqrt(Util.SqrDist(Util.GetRow(x, i), Util.GetRow(xt, i)));
-        k = Math.PI * k / p;
-        k = Math.Sin(k) / l;
-        k = k * k;
-        cov[i] = sf2 * Math.Exp(-2.0 * k);
+          }
+        default: {
+            throw new ArgumentException("CovariancePeriodic only has three hyperparameters.", "k");
+          }
       }
-      return cov;
-    }
-
-    public double[] GetGradient(int i, int j) {
-
-      var res = new double[3];
-      double k = sd[i, j];
-      k = Math.PI * k / p;
-      {
-        double newK = Math.Sin(k) / l;
-        newK = newK * newK;
-        res[0] = 4 * sf2 * Math.Exp(-2 * newK) * newK;
-      }
-      {
-        double r = Math.Sin(k) / l;
-        res[1] = 4 * sf2 / l * Math.Exp(-2 * r * r) * r * Math.Cos(k) * k;
-      }
-      {
-        double newK = Math.Sin(k) / l;
-        newK = newK * newK;
-        res[2] = 2 * sf2 * Math.Exp(-2 * newK);
-      }
-
-      return res;
     }
 
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceProd.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceProd.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceProd.cs	(revision 8477)
@@ -20,4 +20,6 @@
 #endregion
 
+using System;
+using System.Collections.Generic;
 using System.Linq;
 using HeuristicLab.Common;
@@ -48,4 +50,5 @@
       this.factors = cloner.Clone(original.factors);
       this.numberOfVariables = original.numberOfVariables;
+      AttachEventHandlers();
     }
 
@@ -53,4 +56,13 @@
       : base() {
       this.factors = new ItemList<ICovarianceFunction>();
+      AttachEventHandlers();
+    }
+
+    private void AttachEventHandlers() {
+      this.factors.CollectionReset += (sender, args) => ClearCache();
+      this.factors.ItemsAdded += (sender, args) => ClearCache();
+      this.factors.ItemsRemoved += (sender, args) => ClearCache();
+      this.factors.ItemsReplaced += (sender, args) => ClearCache();
+      this.factors.ItemsMoved += (sender, args) => ClearCache();
     }
 
@@ -86,6 +98,35 @@
     }
 
-    public double[] GetGradient(int i, int j) {
-      return factors.Select(t => t.GetGradient(i, j)).SelectMany(seq => seq).ToArray();
+    private Dictionary<int, Tuple<int, int>> cachedParameterMap;
+    public double GetGradient(int i, int j, int k) {
+      if (cachedParameterMap == null) {
+        CalculateParameterMap();
+      }
+      int ti = cachedParameterMap[k].Item1;
+      k = cachedParameterMap[k].Item2;
+      double res = 1.0;
+      for (int ii = 0; ii < factors.Count; ii++) {
+        var f = factors[ii];
+        if (ii == ti) {
+          res *= f.GetGradient(i, j, k);
+        } else {
+          res *= f.GetCovariance(i, j);
+        }
+      }
+      return res;
+    }
+
+    private void ClearCache() {
+      cachedParameterMap = null;
+    }
+
+    private void CalculateParameterMap() {
+      cachedParameterMap = new Dictionary<int, Tuple<int, int>>();
+      int k = 0;
+      for (int ti = 0; ti < factors.Count; ti++) {
+        for (int ti_k = 0; ti_k < factors[ti].GetNumberOfParameters(numberOfVariables); ti_k++) {
+          cachedParameterMap[k++] = Tuple.Create(ti, ti_k);
+        }
+      }
     }
   }
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceRQiso.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceRQiso.cs	(revision 8477)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceRQiso.cs	(revision 8477)
@@ -0,0 +1,145 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "CovarianceRQiso",
+    Description = "Isotropic rational quadratic covariance function for Gaussian processes.")]
+  public class CovarianceRQiso : Item, ICovarianceFunction {
+    [Storable]
+    private double[,] x;
+    [Storable]
+    private double[,] xt;
+    [Storable]
+    private double sf2;
+    public double Scale { get { return sf2; } }
+    [Storable]
+    private double l;
+    public double Length { get { return l; } }
+    [Storable]
+    private double alpha;
+    public double Shape { get { return alpha; } }
+    [Storable]
+    private bool symmetric;
+    private double[,] d2;
+
+    [StorableConstructor]
+    protected CovarianceRQiso(bool deserializing)
+      : base(deserializing) {
+    }
+
+    protected CovarianceRQiso(CovarianceRQiso original, Cloner cloner)
+      : base(original, cloner) {
+      if (original.x != null) {
+        this.x = new double[original.x.GetLength(0), original.x.GetLength(1)];
+        Array.Copy(original.x, this.x, x.Length);
+
+        this.xt = new double[original.xt.GetLength(0), original.xt.GetLength(1)];
+        Array.Copy(original.xt, this.xt, xt.Length);
+
+        this.d2 = new double[original.d2.GetLength(0), original.d2.GetLength(1)];
+        Array.Copy(original.d2, this.d2, d2.Length);
+        this.sf2 = original.sf2;
+      }
+      this.sf2 = original.sf2;
+      this.l = original.l;
+      this.alpha = original.alpha;
+      this.symmetric = original.symmetric;
+    }
+
+    public CovarianceRQiso()
+      : base() {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceRQiso(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return 3;
+    }
+
+    public void SetParameter(double[] hyp) {
+      this.l = Math.Exp(hyp[0]);
+      this.sf2 = Math.Exp(2 * hyp[1]);
+      this.alpha = Math.Exp(hyp[2]);
+      d2 = null;
+    }
+    public void SetData(double[,] x) {
+      SetData(x, x);
+      this.symmetric = true;
+    }
+
+
+    public void SetData(double[,] x, double[,] xt) {
+      this.symmetric = false;
+      this.x = x;
+      this.xt = xt;
+      d2 = null;
+    }
+
+    public double GetCovariance(int i, int j) {
+      if (d2 == null) CalculateSquaredDistances();
+      return sf2 * Math.Pow(1 + 0.5 * d2[i, j] / alpha, -alpha);
+    }
+
+    public double GetGradient(int i, int j, int k) {
+      switch (k) {
+        case 0: return sf2 * Math.Pow(1 + 0.5 * d2[i, j] / alpha, -alpha - 1) * d2[i, j];
+        case 1: return 2 * sf2 * Math.Pow((1 + 0.5 * d2[i, j] / alpha), (-alpha));
+        case 2: {
+            double g = (1 + 0.5 * d2[i, j] / alpha);
+            g = sf2 * Math.Pow(g, -alpha) * (0.5 * d2[i, j] / g - alpha * Math.Log(g));
+            return g;
+          }
+        default: throw new ArgumentException("CovarianceRQiso has three hyperparameters", "k");
+      }
+    }
+
+    private void CalculateSquaredDistances() {
+      if (x.GetLength(1) != xt.GetLength(1)) throw new InvalidOperationException();
+      int rows = x.GetLength(0);
+      int cols = xt.GetLength(0);
+      d2 = new double[rows, cols];
+      double lInv = 1.0 / l;
+      if (symmetric) {
+        for (int i = 0; i < rows; i++) {
+          for (int j = i; j < rows; j++) {
+            d2[i, j] = Util.SqrDist(Util.GetRow(x, i).Select(e => e * lInv), Util.GetRow(xt, j).Select(e => e * lInv));
+            d2[j, i] = d2[i, j];
+          }
+        }
+      } else {
+        for (int i = 0; i < rows; i++) {
+          for (int j = 0; j < cols; j++) {
+            d2[i, j] = Util.SqrDist(Util.GetRow(x, i).Select(e => e * lInv), Util.GetRow(xt, j).Select(e => e * lInv));
+          }
+        }
+      }
+    }
+  }
+}
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceSEard.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceSEard.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceSEard.cs	(revision 8477)
@@ -36,6 +36,16 @@
     [Storable]
     private double sf2;
+    public double Scale { get { return sf2; } }
+
     [Storable]
     private double[] l;
+    public double[] Length {
+      get {
+        if (l == null) return new double[0];
+        var copy = new double[l.Length];
+        Array.Copy(l, copy, copy.Length);
+        return copy;
+      }
+    }
 
     private double[,] sd;
@@ -76,5 +86,5 @@
       this.l = hyp.Take(hyp.Length - 1).Select(Math.Exp).ToArray();
       this.sf2 = Math.Exp(2 * hyp[hyp.Length - 1]);
-      sf2 = Math.Min(10E6, sf2); // upper limit for the scale
+      // sf2 = Math.Min(10E6, sf2); // upper limit for the scale
 
       sd = null;
@@ -99,13 +109,13 @@
     }
 
-    public double[] GetGradient(int i, int j) {
-      var res = new double[l.Length + 1];
-      for (int k = 0; k < l.Length; k++) {
+    public double GetGradient(int i, int j, int k) {
+      if (k < l.Length) {
         double sqrDist = Util.SqrDist(x[i, k] / l[k], xt[j, k] / l[k]);
-
-        res[k] = sf2 * Math.Exp(-sd[i, j] / 2.0) * sqrDist;
+        return sf2 * Math.Exp(-sd[i, j] / 2.0) * sqrDist;
+      } else if (k == l.Length) {
+        return 2.0 * sf2 * Math.Exp(-sd[i, j] / 2.0);
+      } else {
+        throw new ArgumentException("CovarianceSEard has dimension+1 hyperparameters.", "k");
       }
-      res[res.Length - 1] = 2.0 * sf2 * Math.Exp(-sd[i, j] / 2.0);
-      return res;
     }
 
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceSEiso.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceSEiso.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceSEiso.cs	(revision 8477)
@@ -37,6 +37,8 @@
     [Storable]
     private double sf2;
+    public double Scale { get { return sf2; } }
     [Storable]
     private double l;
+    public double Length { get { return l; } }
     [Storable]
     private bool symmetric;
@@ -80,5 +82,5 @@
     public void SetParameter(double[] hyp) {
       this.l = Math.Exp(hyp[0]);
-      this.sf2 = Math.Min(1E6, Math.Exp(2 * hyp[1])); // upper limit for scale
+      this.sf2 = Math.Exp(2 * hyp[1]);
       sd = null;
     }
@@ -101,9 +103,10 @@
     }
 
-    public double[] GetGradient(int i, int j) {
-      var res = new double[2];
-      res[0] = sf2 * Math.Exp(-sd[i, j] / 2.0) * sd[i, j];
-      res[1] = 2.0 * sf2 * Math.Exp(-sd[i, j] / 2.0);
-      return res;
+    public double GetGradient(int i, int j, int k) {
+      switch (k) {
+        case 0: return sf2 * Math.Exp(-sd[i, j] / 2.0) * sd[i, j];
+        case 1: return 2.0 * sf2 * Math.Exp(-sd[i, j] / 2.0);
+        default: throw new ArgumentException("CovarianceSEiso has two hyperparameters", "k");
+      }
     }
 
@@ -113,8 +116,9 @@
       int cols = xt.GetLength(0);
       sd = new double[rows, cols];
+      double lInv = 1.0 / l;
       if (symmetric) {
         for (int i = 0; i < rows; i++) {
           for (int j = i; j < rows; j++) {
-            sd[i, j] = Util.SqrDist(Util.GetRow(x, i).Select(e => e / l), Util.GetRow(xt, j).Select(e => e / l));
+            sd[i, j] = Util.SqrDist(Util.GetRow(x, i).Select(e => e * lInv), Util.GetRow(xt, j).Select(e => e * lInv));
             sd[j, i] = sd[i, j];
           }
@@ -123,5 +127,5 @@
         for (int i = 0; i < rows; i++) {
           for (int j = 0; j < cols; j++) {
-            sd[i, j] = Util.SqrDist(Util.GetRow(x, i).Select(e => e / l), Util.GetRow(xt, j).Select(e => e / l));
+            sd[i, j] = Util.SqrDist(Util.GetRow(x, i).Select(e => e * lInv), Util.GetRow(xt, j).Select(e => e * lInv));
           }
         }
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceSum.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceSum.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceSum.cs	(revision 8477)
@@ -20,4 +20,6 @@
 #endregion
 
+using System;
+using System.Collections.Generic;
 using System.Linq;
 using HeuristicLab.Common;
@@ -48,4 +50,5 @@
       this.terms = cloner.Clone(original.terms);
       this.numberOfVariables = original.numberOfVariables;
+      AttachEventHandlers();
     }
 
@@ -53,4 +56,13 @@
       : base() {
       this.terms = new ItemList<ICovarianceFunction>();
+      AttachEventHandlers();
+    }
+
+    private void AttachEventHandlers() {
+      this.terms.CollectionReset += (sender, args) => ClearCache();
+      this.terms.ItemsAdded += (sender, args) => ClearCache();
+      this.terms.ItemsRemoved += (sender, args) => ClearCache();
+      this.terms.ItemsReplaced += (sender, args) => ClearCache();
+      this.terms.ItemsMoved += (sender, args) => ClearCache();
     }
 
@@ -86,6 +98,25 @@
     }
 
-    public double[] GetGradient(int i, int j) {
-      return terms.Select(t => t.GetGradient(i, j)).SelectMany(seq => seq).ToArray();
+    private Dictionary<int, Tuple<int, int>> cachedParameterMap;
+    public double GetGradient(int i, int j, int k) {
+      if (cachedParameterMap == null) {
+        CalculateParameterMap();
+      }
+      int ti = cachedParameterMap[k].Item1;
+      k = cachedParameterMap[k].Item2;
+      return terms[ti].GetGradient(i, j, k);
+    }
+    private void ClearCache() {
+      cachedParameterMap = null;
+    }
+
+    private void CalculateParameterMap() {
+      cachedParameterMap = new Dictionary<int, Tuple<int, int>>();
+      int k = 0;
+      for (int ti = 0; ti < terms.Count; ti++) {
+        for (int ti_k = 0; ti_k < terms[ti].GetNumberOfParameters(numberOfVariables); ti_k++) {
+          cachedParameterMap[k++] = Tuple.Create(ti, ti_k);
+        }
+      }
     }
   }
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessHyperparameterInitializer.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessHyperparameterInitializer.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessHyperparameterInitializer.cs	(revision 8477)
@@ -38,4 +38,5 @@
     private const string ProblemDataParameterName = "ProblemData";
     private const string HyperparameterParameterName = "Hyperparameter";
+    private const string RandomParameterName = "Random";
 
     #region Parameter Properties
@@ -49,4 +50,7 @@
     public ILookupParameter<IDataAnalysisProblemData> ProblemDataParameter {
       get { return (ILookupParameter<IDataAnalysisProblemData>)Parameters[ProblemDataParameterName]; }
+    }
+    public ILookupParameter<IRandom> RandomParameter {
+      get { return (ILookupParameter<IRandom>)Parameters[RandomParameterName]; }
     }
     // out
@@ -71,4 +75,5 @@
       Parameters.Add(new LookupParameter<ICovarianceFunction>(CovarianceFunctionParameterName, "The covariance function for the Gaussian process model."));
       Parameters.Add(new LookupParameter<IDataAnalysisProblemData>(ProblemDataParameterName, "The input data for the Gaussian process."));
+      Parameters.Add(new LookupParameter<IRandom>(RandomParameterName, "The pseudo random number generator to use for initializing the hyperparameter vector."));
       // out
       Parameters.Add(new LookupParameter<RealVector>(HyperparameterParameterName, "The initial hyperparameter vector for the Gaussian process model."));
@@ -83,5 +88,10 @@
       int l = 1 + MeanFunction.GetNumberOfParameters(inputVariablesCount) +
               CovarianceFunction.GetNumberOfParameters(inputVariablesCount);
-      HyperparameterParameter.ActualValue = new RealVector(l);
+      var r = new RealVector(l);
+      var rand = RandomParameter.ActualValue;
+      for (int i = 0; i < r.Length; i++)
+        r[i] = rand.NextDouble() * 2 - 1;
+
+      HyperparameterParameter.ActualValue = r;
       return base.Apply();
     }
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs	(revision 8477)
@@ -73,5 +73,5 @@
     private double[,] x;
     [Storable]
-    private Scaling scaling;
+    private Scaling inputScaling;
 
 
@@ -82,5 +82,5 @@
       this.meanFunction = cloner.Clone(original.meanFunction);
       this.covarianceFunction = cloner.Clone(original.covarianceFunction);
-      this.scaling = cloner.Clone(original.scaling);
+      this.inputScaling = cloner.Clone(original.inputScaling);
       this.negativeLogLikelihood = original.negativeLogLikelihood;
       this.targetVariable = original.targetVariable;
@@ -103,14 +103,13 @@
       this.allowedInputVariables = allowedInputVariables.ToArray();
 
-      sqrSigmaNoise = Math.Exp(2.0 * hyp.First());
-      sqrSigmaNoise = Math.Max(10E-6, sqrSigmaNoise); // lower limit for the noise level
 
       int nVariables = this.allowedInputVariables.Length;
-      this.meanFunction.SetParameter(hyp.Skip(1)
+      this.meanFunction.SetParameter(hyp
         .Take(this.meanFunction.GetNumberOfParameters(nVariables))
         .ToArray());
-      this.covarianceFunction.SetParameter(hyp.Skip(1 + this.meanFunction.GetNumberOfParameters(nVariables))
+      this.covarianceFunction.SetParameter(hyp.Skip(this.meanFunction.GetNumberOfParameters(nVariables))
         .Take(this.covarianceFunction.GetNumberOfParameters(nVariables))
         .ToArray());
+      sqrSigmaNoise = Math.Exp(2.0 * hyp.Last());
 
       CalculateModel(ds, rows);
@@ -118,8 +117,7 @@
 
     private void CalculateModel(Dataset ds, IEnumerable<int> rows) {
-      scaling = new Scaling(ds, allowedInputVariables, rows);
-      x = AlglibUtil.PrepareAndScaleInputMatrix(ds, allowedInputVariables, rows, scaling);
-
-      var y = ds.GetDoubleValues(targetVariable, rows).ToArray();
+      inputScaling = new Scaling(ds, allowedInputVariables, rows);
+      x = AlglibUtil.PrepareAndScaleInputMatrix(ds, allowedInputVariables, rows, inputScaling);
+      var y = ds.GetDoubleValues(targetVariable, rows);
 
       int n = x.GetLength(0);
@@ -132,5 +130,4 @@
       double[] m = meanFunction.GetMean(x);
       for (int i = 0; i < n; i++) {
-
         for (int j = i; j < n; j++) {
           l[j, i] = covarianceFunction.GetCovariance(i, j) / sqrSigmaNoise;
@@ -144,5 +141,5 @@
 
       var res = alglib.trfac.spdmatrixcholesky(ref l, n, false);
-      if (!res) throw new InvalidOperationException("Matrix is not positive semidefinite");
+      if (!res) throw new ArgumentException("Matrix is not positive semidefinite");
 
       // calculate sum of diagonal elements for likelihood
@@ -162,19 +159,18 @@
       int n = x.GetLength(0);
       int nAllowedVariables = x.GetLength(1);
-      double[,] q = new double[n, n];
-      double[,] eye = new double[n, n];
-      for (int i = 0; i < n; i++) eye[i, i] = 1.0;
 
       int info;
-      alglib.densesolverreport denseSolveRep;
-
-      alglib.spdmatrixcholeskysolvem(l, n, false, eye, n, out info, out denseSolveRep, out q);
-      // double[,] a2 = outerProd(alpha, alpha);
+      alglib.matinvreport matInvRep;
+      double[,] lCopy = new double[l.GetLength(0), l.GetLength(1)];
+      Array.Copy(l, lCopy, lCopy.Length);
+
+      alglib.spdmatrixcholeskyinverse(ref lCopy, n, false, out info, out matInvRep);
+      if (info != 1) throw new ArgumentException("Can't invert matrix to calculate gradients.");
       for (int i = 0; i < n; i++) {
-        for (int j = 0; j < n; j++)
-          q[i, j] = q[i, j] / sqrSigmaNoise - alpha[i] * alpha[j]; // a2[i, j];
-      }
-
-      double noiseGradient = sqrSigmaNoise * Enumerable.Range(0, n).Select(i => q[i, i]).Sum();
+        for (int j = 0; j <= i; j++)
+          lCopy[i, j] = lCopy[i, j] / sqrSigmaNoise - alpha[i] * alpha[j];
+      }
+
+      double noiseGradient = sqrSigmaNoise * Enumerable.Range(0, n).Select(i => lCopy[i, i]).Sum();
 
       double[] meanGradients = new double[meanFunction.GetNumberOfParameters(nAllowedVariables)];
@@ -187,17 +183,17 @@
       if (covGradients.Length > 0) {
         for (int i = 0; i < n; i++) {
-          for (int j = 0; j < n; j++) {
-            var covDeriv = covarianceFunction.GetGradient(i, j);
-            for (int k = 0; k < covGradients.Length; k++) {
-              covGradients[k] += q[i, j] * covDeriv[k];
+          for (int k = 0; k < covGradients.Length; k++) {
+            for (int j = 0; j < i; j++) {
+              covGradients[k] += lCopy[i, j] * covarianceFunction.GetGradient(i, j, k);
             }
+            covGradients[k] += 0.5 * lCopy[i, i] * covarianceFunction.GetGradient(i, i, k);
           }
         }
-        covGradients = covGradients.Select(g => g / 2.0).ToArray();
-      }
-
-      return new double[] { noiseGradient }
-        .Concat(meanGradients)
-        .Concat(covGradients).ToArray();
+      }
+
+      return
+        meanGradients
+        .Concat(covGradients)
+        .Concat(new double[] { noiseGradient }).ToArray();
     }
 
@@ -220,5 +216,5 @@
 
     private IEnumerable<double> GetEstimatedValuesHelper(Dataset dataset, IEnumerable<int> rows) {
-      var newX = AlglibUtil.PrepareAndScaleInputMatrix(dataset, allowedInputVariables, rows, scaling);
+      var newX = AlglibUtil.PrepareAndScaleInputMatrix(dataset, allowedInputVariables, rows, inputScaling);
       int newN = newX.GetLength(0);
       int n = x.GetLength(0);
@@ -230,5 +226,5 @@
       // var kss = new double[newN];
       var Ks = new double[newN, n];
-      double[,] sWKs = new double[n, newN];
+      //double[,] sWKs = new double[n, newN];
       // double[,] v;
 
@@ -242,8 +238,7 @@
       var ms = meanFunction.GetMean(newX);
       for (int i = 0; i < newN; i++) {
-
         for (int j = 0; j < n; j++) {
           Ks[i, j] = covarianceFunction.GetCovariance(j, i);
-          sWKs[j, i] = Ks[i, j] / Math.Sqrt(sqrSigmaNoise);
+          //sWKs[j, i] = Ks[i, j] / Math.Sqrt(sqrSigmaNoise);
         }
       }
@@ -252,12 +247,47 @@
       // alglib.rmatrixsolvem(l, n, sWKs, newN, true, out info, out denseSolveRep, out v);
 
-
+      return Enumerable.Range(0, newN)
+        .Select(i => ms[i] + Util.ScalarProd(Util.GetRow(Ks, i), alpha));
+      //for (int i = 0; i < newN; i++) {
+      //  // predMean[i] = ms[i] + prod(GetRow(Ks, i), alpha);
+      //  // var sumV2 = prod(GetCol(v, i), GetCol(v, i));
+      //  // predVar[i] = kss[i] - sumV2;
+      //}
+
+    }
+
+    public IEnumerable<double> GetEstimatedVariance(Dataset dataset, IEnumerable<int> rows) {
+      var newX = AlglibUtil.PrepareAndScaleInputMatrix(dataset, allowedInputVariables, rows, inputScaling);
+      int newN = newX.GetLength(0);
+      int n = x.GetLength(0);
+
+      var kss = new double[newN];
+      double[,] sWKs = new double[n, newN];
+
+      // for stddev 
+      covarianceFunction.SetData(newX);
+      for (int i = 0; i < newN; i++)
+        kss[i] = covarianceFunction.GetCovariance(i, i);
+
+      covarianceFunction.SetData(x, newX);
       for (int i = 0; i < newN; i++) {
-        // predMean[i] = ms[i] + prod(GetRow(Ks, i), alpha);
-        yield return ms[i] + Util.ScalarProd(Util.GetRow(Ks, i), alpha);
-        // var sumV2 = prod(GetCol(v, i), GetCol(v, i));
-        // predVar[i] = kss[i] - sumV2;
-      }
-
+        for (int j = 0; j < n; j++) {
+          sWKs[j, i] = covarianceFunction.GetCovariance(j, i) / Math.Sqrt(sqrSigmaNoise);
+        }
+      }
+
+      // for stddev 
+      int info;
+      alglib.densesolverreport denseSolveRep;
+      double[,] v;
+
+      alglib.rmatrixsolvem(l, n, sWKs, newN, false, out info, out denseSolveRep, out v);
+
+      for (int i = 0; i < newN; i++) {
+        var sumV = Util.ScalarProd(Util.GetCol(v, i), Util.GetCol(v, i));
+        kss[i] -= sumV;
+        if (kss[i] < 0) kss[i] = 0;
+      }
+      return kss;
     }
   }
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegression.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegression.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegression.cs	(revision 8477)
@@ -55,4 +55,6 @@
     private const string MinimizationIterationsParameterName = "Iterations";
     private const string ApproximateGradientsParameterName = "ApproximateGradients";
+    private const string SeedParameterName = "Seed";
+    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
 
     #region parameter properties
@@ -65,4 +67,10 @@
     public IValueParameter<IntValue> MinimizationIterationsParameter {
       get { return (IValueParameter<IntValue>)Parameters[MinimizationIterationsParameterName]; }
+    }
+    public IValueParameter<IntValue> SeedParameter {
+      get { return (IValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+    public IValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (IValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
     }
     #endregion
@@ -80,5 +88,8 @@
       get { return MinimizationIterationsParameter.Value.Value; }
     }
+    public int Seed { get { return SeedParameter.Value.Value; } set { SeedParameter.Value.Value = value; } }
+    public bool SetSeedRandomly { get { return SetSeedRandomlyParameter.Value.Value; } set { SetSeedRandomlyParameter.Value.Value = value; } }
     #endregion
+
     [StorableConstructor]
     private GaussianProcessRegression(bool deserializing) : base(deserializing) { }
@@ -97,11 +108,15 @@
 
       Parameters.Add(new ConstrainedValueParameter<IMeanFunction>(MeanFunctionParameterName, "The mean function to use.",
-        new ItemSet<IMeanFunction>(meanFunctions), meanFunctions.First()));
+        new ItemSet<IMeanFunction>(meanFunctions), meanFunctions.OfType<MeanConst>().First()));
       Parameters.Add(new ConstrainedValueParameter<ICovarianceFunction>(CovarianceFunctionParameterName, "The covariance function to use.",
-        new ItemSet<ICovarianceFunction>(covFunctions), covFunctions.First()));
+        new ItemSet<ICovarianceFunction>(covFunctions), covFunctions.OfType<CovarianceSEiso>().First()));
       Parameters.Add(new ValueParameter<IntValue>(MinimizationIterationsParameterName, "The number of iterations for likelihood optimization with LM-BFGS.", new IntValue(20)));
+      Parameters.Add(new ValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+      Parameters.Add(new ValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+
       Parameters.Add(new ValueParameter<BoolValue>(ApproximateGradientsParameterName, "Indicates that gradients should not be approximated (necessary for LM-BFGS).", new BoolValue(false)));
       Parameters[ApproximateGradientsParameterName].Hidden = true; // should not be changed
 
+      var randomCreator = new HeuristicLab.Random.RandomCreator();
       var gpInitializer = new GaussianProcessHyperparameterInitializer();
       var bfgsInitializer = new LbfgsInitializer();
@@ -115,5 +130,10 @@
       var solutionCreator = new GaussianProcessRegressionSolutionCreator();
 
-      OperatorGraph.InitialOperator = gpInitializer;
+      OperatorGraph.InitialOperator = randomCreator;
+      randomCreator.SeedParameter.ActualName = SeedParameterName;
+      randomCreator.SeedParameter.Value = null;
+      randomCreator.SetSeedRandomlyParameter.ActualName = SetSeedRandomlyParameterName;
+      randomCreator.SetSeedRandomlyParameter.Value = null;
+      randomCreator.Successor = gpInitializer;
 
       gpInitializer.CovarianceFunctionParameter.ActualName = CovarianceFunctionParameterName;
@@ -121,4 +141,5 @@
       gpInitializer.ProblemDataParameter.ActualName = Problem.ProblemDataParameter.Name;
       gpInitializer.HyperparameterParameter.ActualName = modelCreator.HyperparameterParameter.Name;
+      gpInitializer.RandomParameter.ActualName = randomCreator.RandomParameter.Name;
       gpInitializer.Successor = bfgsInitializer;
 
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs	(revision 8477)
@@ -20,4 +20,5 @@
 #endregion
 
+using System;
 using System.Linq;
 using HeuristicLab.Common;
@@ -60,8 +61,15 @@
 
     public override IOperation Apply() {
-      var model = Create(ProblemData, Hyperparameter.ToArray(), MeanFunction, CovarianceFunction);
-      ModelParameter.ActualValue = model;
-      NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(model.NegativeLogLikelihood);
-      HyperparameterGradientsParameter.ActualValue = new RealVector(model.GetHyperparameterGradients());
+      try {
+        var model = Create(ProblemData, Hyperparameter.ToArray(), MeanFunction, CovarianceFunction);
+        ModelParameter.ActualValue = model;
+        NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(model.NegativeLogLikelihood);
+        HyperparameterGradientsParameter.ActualValue = new RealVector(model.GetHyperparameterGradients());
+        return base.Apply();
+      }
+      catch (ArgumentException) { }
+      catch (alglib.alglibexception) { }
+      NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(1E300);
+      HyperparameterGradientsParameter.ActualValue = new RealVector(Hyperparameter.Count());
       return base.Apply();
     }
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolution.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolution.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolution.cs	(revision 8477)
@@ -20,4 +20,6 @@
 #endregion
 
+using System.Collections.Generic;
+using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -51,4 +53,18 @@
       return new GaussianProcessRegressionSolution(this, cloner);
     }
+
+    public IEnumerable<double> EstimatedVariance {
+      get { return GetEstimatedVariance(Enumerable.Range(0, ProblemData.Dataset.Rows)); }
+    }
+    public IEnumerable<double> EstimatedTrainingVariance {
+      get { return GetEstimatedVariance(ProblemData.TrainingIndices); }
+    }
+    public IEnumerable<double> EstimatedTestVariance {
+      get { return GetEstimatedVariance(ProblemData.TestIndices); }
+    }
+
+    public IEnumerable<double> GetEstimatedVariance(IEnumerable<int> rows) {
+      return Model.GetEstimatedVariance(ProblemData.Dataset, rows);
+    }
   }
 }
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs	(revision 8477)
@@ -75,6 +75,6 @@
 
     public override IOperation Apply() {
-      var m = ModelParameter.ActualValue;
-      var data = ProblemDataParameter.ActualValue;
+      var m = (IGaussianProcessModel)ModelParameter.ActualValue.Clone();
+      var data = (IRegressionProblemData)ProblemDataParameter.ActualValue.Clone();
       var s = new GaussianProcessRegressionSolution(m, data);
 
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/ICovarianceFunction.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/ICovarianceFunction.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/ICovarianceFunction.cs	(revision 8477)
@@ -30,5 +30,5 @@
 
     double GetCovariance(int i, int j);
-    double[] GetGradient(int i, int j);
+    double GetGradient(int i, int j, int k);
   }
 }
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanConst.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanConst.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanConst.cs	(revision 8477)
@@ -32,4 +32,6 @@
     [Storable]
     private double c;
+    public double Value { get { return c; } }
+
     public int GetNumberOfParameters(int numberOfVariables) {
       return 1;
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanLinear.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanLinear.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanLinear.cs	(revision 8477)
@@ -31,4 +31,12 @@
     [Storable]
     private double[] alpha;
+    public double[] Weights {
+      get {
+        if (alpha == null) return new double[0];
+        var copy = new double[alpha.Length];
+        Array.Copy(alpha, copy, copy.Length);
+        return copy;
+      }
+    }
     public int GetNumberOfParameters(int numberOfVariables) {
       return numberOfVariables;
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanProd.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanProd.cs	(revision 8477)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanProd.cs	(revision 8477)
@@ -0,0 +1,109 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "MeanProd", Description = "Product of mean functions for Gaussian processes.")]
+  public class MeanProd : Item, IMeanFunction {
+    [Storable]
+    private ItemList<IMeanFunction> factors;
+
+    [Storable]
+    private int numberOfVariables;
+
+    public ItemList<IMeanFunction> Factors {
+      get { return factors; }
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      this.numberOfVariables = numberOfVariables;
+      return factors.Select(t => t.GetNumberOfParameters(numberOfVariables)).Sum();
+    }
+
+    [StorableConstructor]
+    protected MeanProd(bool deserializing)
+      : base(deserializing) {
+    }
+
+    protected MeanProd(MeanProd original, Cloner cloner)
+      : base(original, cloner) {
+      this.factors = cloner.Clone(original.factors);
+      this.numberOfVariables = original.numberOfVariables;
+    }
+
+    public MeanProd() {
+      this.factors = new ItemList<IMeanFunction>();
+    }
+
+    public void SetParameter(double[] hyp) {
+      int offset = 0;
+      foreach (var t in factors) {
+        var numberOfParameters = t.GetNumberOfParameters(numberOfVariables);
+        t.SetParameter(hyp.Skip(offset).Take(numberOfParameters).ToArray());
+        offset += numberOfParameters;
+      }
+    }
+
+    public void SetData(double[,] x) {
+      foreach (var t in factors) t.SetData(x);
+    }
+
+    public double[] GetMean(double[,] x) {
+      var res = factors.First().GetMean(x);
+      foreach (var t in factors.Skip(1)) {
+        var a = t.GetMean(x);
+        for (int i = 0; i < res.Length; i++) res[i] *= a[i];
+      }
+      return res;
+    }
+
+    public double[] GetGradients(int k, double[,] x) {
+      double[] res = Enumerable.Repeat(1.0, x.GetLength(0)).ToArray();
+      // find index of factor for the given k
+      int j = 0;
+      while (k >= factors[j].GetNumberOfParameters(numberOfVariables)) {
+        k -= factors[j].GetNumberOfParameters(numberOfVariables);
+        j++;
+      }
+      for (int i = 0; i < factors.Count; i++) {
+        var f = factors[i];
+        if (i == j) {
+          // multiply gradient
+          var g = f.GetGradients(k, x);
+          for (int ii = 0; ii < res.Length; ii++) res[ii] *= g[ii];
+        } else {
+          // multiply mean
+          var m = f.GetMean(x);
+          for (int ii = 0; ii < res.Length; ii++) res[ii] *= m[ii];
+        }
+      }
+      return res;
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanProd(this, cloner);
+    }
+  }
+}
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanSum.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanSum.cs	(revision 8477)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanSum.cs	(revision 8477)
@@ -0,0 +1,89 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "MeanSum", Description = "Sum of mean functions for Gaussian processes.")]
+  public class MeanSum : Item, IMeanFunction {
+    [Storable]
+    private ItemList<IMeanFunction> terms;
+
+    [Storable]
+    private int numberOfVariables;
+    public ItemList<IMeanFunction> Terms {
+      get { return terms; }
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      this.numberOfVariables = numberOfVariables;
+      return terms.Select(t => t.GetNumberOfParameters(numberOfVariables)).Sum();
+    }
+    [StorableConstructor]
+    protected MeanSum(bool deserializing) : base(deserializing) { }
+    protected MeanSum(MeanSum original, Cloner cloner)
+      : base(original, cloner) {
+      this.terms = cloner.Clone(original.terms);
+      this.numberOfVariables = original.numberOfVariables;
+    }
+    public MeanSum() {
+      this.terms = new ItemList<IMeanFunction>();
+    }
+
+    public void SetParameter(double[] hyp) {
+      int offset = 0;
+      foreach (var t in terms) {
+        var numberOfParameters = t.GetNumberOfParameters(numberOfVariables);
+        t.SetParameter(hyp.Skip(offset).Take(numberOfParameters).ToArray());
+        offset += numberOfParameters;
+      }
+    }
+
+    public void SetData(double[,] x) {
+      foreach (var t in terms) t.SetData(x);
+    }
+
+    public double[] GetMean(double[,] x) {
+      var res = terms.First().GetMean(x);
+      foreach (var t in terms.Skip(1)) {
+        var a = t.GetMean(x);
+        for (int i = 0; i < res.Length; i++) res[i] += a[i];
+      }
+      return res;
+    }
+
+    public double[] GetGradients(int k, double[,] x) {
+      int i = 0;
+      while (k >= terms[i].GetNumberOfParameters(numberOfVariables)) {
+        k -= terms[i].GetNumberOfParameters(numberOfVariables);
+        i++;
+      }
+      return terms[i].GetGradients(k, x);
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanSum(this, cloner);
+    }
+  }
+}
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/Util.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/Util.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/Util.cs	(revision 8477)
@@ -20,5 +20,4 @@
 #endregion
 
-using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -32,9 +31,9 @@
     public static double SqrDist(double x, double y) {
       double d = x - y;
-      return Math.Max(d * d, 0.0);
+      return d * d;
     }
 
     public static double SqrDist(IEnumerable<double> x, IEnumerable<double> y) {
-      return x.Zip(y, SqrDist).Sum();
+      return x.Zip(y, (a, b) => (a - b) * (a - b)).Sum();
     }
 
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj	(revision 8477)
@@ -101,6 +101,6 @@
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="ALGLIB-3.5.0, Version=3.5.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
-      <HintPath>..\..\..\..\trunk\sources\bin\ALGLIB-3.5.0.dll</HintPath>
+    <Reference Include="ALGLIB-3.6.0, Version=3.6.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\..\..\trunk\sources\bin\ALGLIB-3.6.0.dll</HintPath>
       <Private>False</Private>
     </Reference>
@@ -174,4 +174,9 @@
     </Compile>
     <Compile Include="FixedDataAnalysisAlgorithm.cs" />
+    <Compile Include="GaussianProcess\CovarianceRQiso.cs" />
+    <Compile Include="GaussianProcess\CovarianceNoise.cs" />
+    <Compile Include="GaussianProcess\CovarianceConst.cs" />
+    <Compile Include="GaussianProcess\MeanProd.cs" />
+    <Compile Include="GaussianProcess\MeanSum.cs" />
     <Compile Include="GaussianProcess\CovarianceProd.cs" />
     <Compile Include="GaussianProcess\CovarianceSum.cs" />
@@ -197,4 +202,6 @@
     <Compile Include="Interfaces\IGaussianProcessModel.cs" />
     <Compile Include="Interfaces\IGaussianProcessSolution.cs" />
+    <Compile Include="Interfaces\INcaClassificationSolution.cs" />
+    <Compile Include="Interfaces\INcaModel.cs" />
     <Compile Include="Interfaces\INearestNeighbourClassificationSolution.cs" />
     <Compile Include="Interfaces\INearestNeighbourRegressionSolution.cs" />
@@ -227,4 +234,12 @@
     <Compile Include="Linear\MultinomialLogitClassificationSolution.cs" />
     <Compile Include="Linear\MultinomialLogitModel.cs" />
+    <Compile Include="Nca\Initialization\INcaInitializer.cs" />
+    <Compile Include="Nca\Initialization\LdaInitializer.cs" />
+    <Compile Include="Nca\Initialization\PcaInitializer.cs" />
+    <Compile Include="Nca\Initialization\RandomInitializer.cs" />
+    <Compile Include="Nca\Matrix.cs" />
+    <Compile Include="Nca\NcaAlgorithm.cs" />
+    <Compile Include="Nca\NcaClassificationSolution.cs" />
+    <Compile Include="Nca\NcaModel.cs" />
     <Compile Include="NearestNeighbour\NearestNeighbourClassification.cs" />
     <Compile Include="NearestNeighbour\NearestNeighbourClassificationSolution.cs" />
@@ -280,4 +295,9 @@
       <Name>HeuristicLab.Problems.DataAnalysis-3.4</Name>
     </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Random\3.3\HeuristicLab.Random-3.3.csproj">
+      <Project>{F4539FB6-4708-40C9-BE64-0A1390AEA197}</Project>
+      <Name>HeuristicLab.Random-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
   </ItemGroup>
   <ItemGroup>
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModel.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModel.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModel.cs	(revision 8477)
@@ -20,4 +20,5 @@
 #endregion
 
+using System.Collections.Generic;
 using HeuristicLab.Problems.DataAnalysis;
 
@@ -31,4 +32,6 @@
     ICovarianceFunction CovarianceFunction { get; }
     double[] GetHyperparameterGradients();
+
+    IEnumerable<double> GetEstimatedVariance(Dataset ds, IEnumerable<int> rows);
   }
 }
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaClassificationSolution.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaClassificationSolution.cs	(revision 8477)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaClassificationSolution.cs	(revision 8477)
@@ -0,0 +1,28 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public interface INcaClassificationSolution : IClassificationSolution {
+    new INcaModel Model { get; }
+  }
+}
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaModel.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaModel.cs	(revision 8477)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaModel.cs	(revision 8477)
@@ -0,0 +1,31 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public interface INcaModel : IClassificationModel {
+    new INcaClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData);
+
+    double[,] Reduce(Dataset dataset, IEnumerable<int> rows);
+  }
+}
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/Scaling.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/Scaling.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/Scaling.cs	(revision 8477)
@@ -59,4 +59,9 @@
       return ds.GetDoubleValues(variable, rows).Select(x => (x - min) / (max - min));
     }
+
+    public void GetScalingParameters(string variable, out double min, out double max) {
+      min = scalingParameters[variable].Item1;
+      max = scalingParameters[variable].Item2;
+    }
   }
 }
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/INcaInitializer.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/INcaInitializer.cs	(revision 8477)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/INcaInitializer.cs	(revision 8477)
@@ -0,0 +1,35 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public interface INCAInitializer : IItem {
+    /// <summary>
+    /// Calculates an initial projection for the NCA to start from.
+    /// </summary>
+    /// <param name="data">The problem data that contains the AllowedInputVariables and TrainingIndices.</param>
+    /// <param name="dimensions">The amount of columns in the matrix</param>
+    /// <returns>A flat representation of a matrix that is read row-wise and contains AllowedInputVariables * TrainingIndices numbers.</returns>
+    double[] Initialize(IClassificationProblemData data, int dimensions);
+  }
+}
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/LdaInitializer.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/LdaInitializer.cs	(revision 8477)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/LdaInitializer.cs	(revision 8477)
@@ -0,0 +1,80 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("LDA", "Initializes the matrix by performing a linear discriminant analysis.")]
+  [StorableClass]
+  public class LDAInitializer : Item, INCAInitializer {
+
+    [StorableConstructor]
+    protected LDAInitializer(bool deserializing) : base(deserializing) { }
+    protected LDAInitializer(LDAInitializer original, Cloner cloner) : base(original, cloner) { }
+    public LDAInitializer() : base() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new LDAInitializer(this, cloner);
+    }
+
+    public double[] Initialize(IClassificationProblemData data, int dimensions) {
+      var instances = data.TrainingIndices.Count();
+      var attributes = data.AllowedInputVariables.Count();
+
+      var ldaDs = new double[instances, attributes + 1];
+      int row, col = 0;
+      foreach (var variable in data.AllowedInputVariables) {
+        row = 0;
+        foreach (var value in data.Dataset.GetDoubleValues(variable, data.TrainingIndices)) {
+          ldaDs[row, col] = value;
+          row++;
+        }
+        col++;
+      }
+      row = 0;
+      var uniqueClasses = new Dictionary<double, int>();
+      foreach (var label in data.Dataset.GetDoubleValues(data.TargetVariable, data.TrainingIndices)) {
+        if (!uniqueClasses.ContainsKey(label))
+          uniqueClasses[label] = uniqueClasses.Count;
+        ldaDs[row++, attributes] = label;
+      }
+      for (row = 0; row < instances; row++)
+        ldaDs[row, attributes] = uniqueClasses[ldaDs[row, attributes]];
+
+      int info;
+      double[,] matrix;
+      alglib.fisherldan(ldaDs, instances, attributes, uniqueClasses.Count, out info, out matrix);
+
+      var result = new double[attributes * dimensions];
+      for (int i = 0; i < attributes; i++)
+        for (int j = 0; j < dimensions; j++)
+          result[i * dimensions + j] = matrix[i, j];
+
+      return result;
+    }
+
+  }
+}
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/PcaInitializer.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/PcaInitializer.cs	(revision 8477)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/PcaInitializer.cs	(revision 8477)
@@ -0,0 +1,71 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("PCA", "Initializes the matrix by performing a principal components analysis.")]
+  [StorableClass]
+  public sealed class PCAInitializer : Item, INCAInitializer {
+
+    [StorableConstructor]
+    private PCAInitializer(bool deserializing) : base(deserializing) { }
+    private PCAInitializer(PCAInitializer original, Cloner cloner) : base(original, cloner) { }
+    public PCAInitializer() : base() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new PCAInitializer(this, cloner);
+    }
+
+    public double[] Initialize(IClassificationProblemData data, int dimensions) {
+      var instances = data.TrainingIndices.Count();
+      var attributes = data.AllowedInputVariables.Count();
+
+      var pcaDs = new double[instances, attributes];
+      int col = 0;
+      foreach (var variable in data.AllowedInputVariables) {
+        int row = 0;
+        foreach (var value in data.Dataset.GetDoubleValues(variable, data.TrainingIndices)) {
+          pcaDs[row, col] = value;
+          row++;
+        }
+        col++;
+      }
+
+      int info;
+      double[] varianceValues;
+      double[,] matrix;
+      alglib.pcabuildbasis(pcaDs, instances, attributes, out info, out varianceValues, out matrix);
+
+      var result = new double[attributes * dimensions];
+      for (int i = 0; i < attributes; i++)
+        for (int j = 0; j < dimensions; j++)
+          result[i * dimensions + j] = matrix[i, j];
+
+      return result;
+    }
+
+  }
+}
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/RandomInitializer.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/RandomInitializer.cs	(revision 8477)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/RandomInitializer.cs	(revision 8477)
@@ -0,0 +1,81 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Random;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("Random", "Initializes the matrix randomly.")]
+  [StorableClass]
+  public class RandomInitializer : ParameterizedNamedItem, INCAInitializer {
+    private IValueParameter<IntValue> RandomParameter {
+      get { return (IValueParameter<IntValue>)Parameters["Seed"]; }
+    }
+    private IValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (IValueParameter<BoolValue>)Parameters["SetSeedRandomly"]; }
+    }
+
+    public int Seed {
+      get { return RandomParameter.Value.Value; }
+      set { RandomParameter.Value.Value = value; }
+    }
+
+    public bool SetSeedRandomly {
+      get { return SetSeedRandomlyParameter.Value.Value; }
+      set { SetSeedRandomlyParameter.Value.Value = value; }
+    }
+
+    [StorableConstructor]
+    protected RandomInitializer(bool deserializing) : base(deserializing) { }
+    protected RandomInitializer(RandomInitializer original, Cloner cloner) : base(original, cloner) { }
+    public RandomInitializer()
+      : base() {
+      Parameters.Add(new ValueParameter<IntValue>("Seed", "The seed for the random number generator.", new IntValue(0)));
+      Parameters.Add(new ValueParameter<BoolValue>("SetSeedRandomly", "Whether the seed should be randomized for each call.", new BoolValue(true)));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RandomInitializer(this, cloner);
+    }
+
+    public double[] Initialize(IClassificationProblemData data, int dimensions) {
+      var instances = data.TrainingIndices.Count();
+      var attributes = data.AllowedInputVariables.Count();
+
+      var random = new MersenneTwister();
+      if (SetSeedRandomly) Seed = random.Next();
+      random.Reset(Seed);
+
+      var range = data.AllowedInputVariables.Select(x => data.Dataset.GetDoubleValues(x).Max() - data.Dataset.GetDoubleValues(x).Min()).ToArray();
+      var matrix = new double[attributes * dimensions];
+      for (int i = 0; i < matrix.Length; i++)
+        matrix[i] = random.NextDouble() / range[i / dimensions];
+
+      return matrix;
+    }
+  }
+}
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Matrix.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Matrix.cs	(revision 8477)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Matrix.cs	(revision 8477)
@@ -0,0 +1,212 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.PluginInfrastructure;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [NonDiscoverableType]
+  internal class Matrix : IEnumerable<double>, IDeepCloneable {
+    // this type is immutable
+    private readonly IEnumerable<double> values;
+    public readonly int Rows;
+    public readonly int Columns;
+
+    protected Matrix(Matrix original, Cloner cloner) {
+      this.values = original.values.ToArray();
+      this.Rows = original.Rows;
+      this.Columns = original.Columns;
+      cloner.RegisterClonedObject(original, this);
+    }
+    public Matrix(IEnumerable<double> vector) {
+      this.values = vector;
+      Rows = 1;
+      Columns = vector.Count();
+    }
+    public Matrix(IEnumerable<double> vector, int length) {
+      this.values = vector;
+      Rows = 1;
+      Columns = length;
+    }
+    public Matrix(double[,] matrix) {
+      this.values = GetOnlineValues(matrix);
+      Rows = matrix.GetLength(0);
+      Columns = matrix.GetLength(1);
+    }
+    public Matrix(IEnumerable<double> matrix, int rows, int columns) {
+      this.values = matrix;
+      Rows = rows;
+      Columns = columns;
+    }
+
+    public object Clone() {
+      return Clone(new Cloner());
+    }
+    public IDeepCloneable Clone(Cloner cloner) {
+      return new Matrix(this, cloner);
+    }
+
+    public Matrix Transpose() {
+      var result = new Matrix(Transpose(values, Columns, Rows), Columns, Rows);
+      return result;
+    }
+
+    private IEnumerable<double> Transpose(IEnumerable<double> values, int rows, int columns) {
+      // vectors don't need to be transposed
+      if (rows == 1 || columns == 1) {
+        foreach (var v in values) yield return v;
+        yield break;
+      }
+
+      int skip = 0;
+      var iter = values.GetEnumerator();
+      if (!iter.MoveNext()) yield break;
+      while (skip < rows) {
+        for (int i = 0; i < skip; i++) iter.MoveNext();
+        yield return iter.Current;
+        for (int j = 0; j < columns - 1; j++) {
+          for (int i = 0; i < rows; i++) iter.MoveNext();
+          yield return iter.Current;
+        }
+        skip++;
+        if (skip < rows) {
+          iter = values.GetEnumerator();
+          iter.MoveNext();
+        }
+      }
+    }
+
+    public Matrix Add(Matrix other) {
+      return new Matrix(AddOnline(other), Rows, Columns);
+    }
+
+    public void AddTo(double[,] matrix) {
+      if (Rows != matrix.GetLength(0) || Columns != matrix.GetLength(1)) throw new ArgumentException("unequal size", "matrix");
+      var iter = values.GetEnumerator();
+      for (int i = 0; i < Rows; i++)
+        for (int j = 0; j < Columns; j++) {
+          iter.MoveNext();
+          matrix[i, j] += iter.Current;
+        }
+    }
+
+    public Matrix Subtract(Matrix other) {
+      return new Matrix(SubtractOnline(other), Rows, Columns);
+    }
+
+    public Matrix Multiply(Matrix other) {
+      return new Matrix(MultiplyOnline(other), Rows, other.Columns);
+    }
+
+    public Matrix Multiply(double value) {
+      return new Matrix(values.Select(x => x * value), Rows, Columns);
+    }
+
+    public double VectorLength() {
+      return Math.Sqrt(SquaredVectorLength());
+    }
+
+    public double SquaredVectorLength() {
+      if (Rows != 1) throw new ArgumentException("Length only works on vectors.");
+      return values.Sum(x => x * x);
+    }
+
+    public Matrix OuterProduct(Matrix other) {
+      if (Rows != 1 || other.Rows != 1) throw new ArgumentException("OuterProduct can only be applied to vectors.");
+      return Transpose().Multiply(other);
+    }
+
+    public Matrix Negate() {
+      return new Matrix(values.Select(x => -x), Rows, Columns);
+    }
+
+    public Matrix Apply() {
+      return new Matrix(values.ToArray(), Rows, Columns);
+    }
+
+    public IEnumerator<double> GetEnumerator() { return values.GetEnumerator(); }
+    IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); }
+
+
+    private IEnumerable<double> AddOnline(Matrix other) {
+      if (Rows != other.Rows || Columns != other.Columns) throw new ArgumentException("Number of rows and columns are not equal.");
+      var meIter = values.GetEnumerator();
+      var otherIter = other.GetEnumerator();
+      if (!meIter.MoveNext()) yield break;
+      if (!otherIter.MoveNext()) yield break;
+      for (int i = 0; i < Rows * Columns; i++) {
+        yield return meIter.Current + otherIter.Current;
+        meIter.MoveNext();
+        otherIter.MoveNext();
+      }
+    }
+
+    private IEnumerable<double> SubtractOnline(Matrix other) {
+      if (Rows != other.Rows || Columns != other.Columns) throw new ArgumentException("Number of rows and columns are not equal.");
+      var meIter = values.GetEnumerator();
+      var otherIter = other.GetEnumerator();
+      if (!meIter.MoveNext()) yield break;
+      if (!otherIter.MoveNext()) yield break;
+      for (int i = 0; i < Rows * Columns; i++) {
+        yield return meIter.Current - otherIter.Current;
+        meIter.MoveNext();
+        otherIter.MoveNext();
+      }
+    }
+
+    private IEnumerable<double> MultiplyOnline(Matrix other) {
+      if (Columns != other.Rows) throw new ArgumentException("Number of rows and columns are not equal.");
+      var meIter = values.GetEnumerator();
+      var otherByColumn = other.Transpose();
+      var otherIter = otherByColumn.GetEnumerator();
+      if (!meIter.MoveNext()) yield break;
+      if (!otherIter.MoveNext()) yield break;
+      for (int r = 0; r < Rows; r++) {
+        var row = new double[Columns];
+        for (int x = 0; x < Columns; x++) {
+          row[x] = meIter.Current;
+          meIter.MoveNext();
+        }
+        for (int c = 0; c < other.Columns; c++) {
+          var sum = 0.0;
+          for (int y = 0; y < other.Rows; y++) {
+            sum += row[y] * otherIter.Current;
+            otherIter.MoveNext();
+          }
+          yield return sum;
+        }
+        otherIter = otherByColumn.GetEnumerator();
+        otherIter.MoveNext();
+      }
+    }
+
+    private IEnumerable<double> GetOnlineValues(double[,] matrix) {
+      for (int i = 0; i < matrix.GetLength(0); i++)
+        for (int j = 0; j < matrix.GetLength(1); j++) {
+          yield return matrix[i, j];
+        }
+    }
+  }
+}
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaAlgorithm.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaAlgorithm.cs	(revision 8477)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaAlgorithm.cs	(revision 8477)
@@ -0,0 +1,255 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Analysis;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.PluginInfrastructure;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  internal delegate void Reporter(double quality, double[] coefficients);
+  /// <summary>
+  /// Neighborhood Components Analysis
+  /// </summary>
+  [Item("Neighborhood Components Analysis (NCA)", "Implementation of Neighborhood Components Analysis based on the description of J. Goldberger, S. Roweis, G. Hinton, R. Salakhutdinov. 2005. Neighbourhood Component Analysis. Advances in Neural Information Processing Systems, 17. pp. 513-520.")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class NcaAlgorithm : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+    #region Parameter Properties
+    public IFixedValueParameter<IntValue> KParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters["K"]; }
+    }
+    public IFixedValueParameter<IntValue> DimensionsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters["Dimensions"]; }
+    }
+    public IConstrainedValueParameter<INCAInitializer> InitializationParameter {
+      get { return (IConstrainedValueParameter<INCAInitializer>)Parameters["Initialization"]; }
+    }
+    public IFixedValueParameter<IntValue> NeighborSamplesParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters["NeighborSamples"]; }
+    }
+    public IFixedValueParameter<IntValue> IterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters["Iterations"]; }
+    }
+    #endregion
+
+    #region Properties
+    private int K {
+      get { return KParameter.Value.Value; }
+      set { KParameter.Value.Value = value; }
+    }
+    private int Dimensions {
+      get { return DimensionsParameter.Value.Value; }
+      set { DimensionsParameter.Value.Value = value; }
+    }
+    private int NeighborSamples {
+      get { return NeighborSamplesParameter.Value.Value; }
+      set { NeighborSamplesParameter.Value.Value = value; }
+    }
+    private int Iterations {
+      get { return IterationsParameter.Value.Value; }
+      set { IterationsParameter.Value.Value = value; }
+    }
+    #endregion
+
+    [StorableConstructor]
+    private NcaAlgorithm(bool deserializing) : base(deserializing) { }
+    private NcaAlgorithm(NcaAlgorithm original, Cloner cloner) : base(original, cloner) { }
+    public NcaAlgorithm()
+      : base() {
+      Parameters.Add(new FixedValueParameter<IntValue>("K", "The K for the nearest neighbor.", new IntValue(1)));
+      Parameters.Add(new FixedValueParameter<IntValue>("Dimensions", "The number of dimensions that NCA should reduce the data to.", new IntValue(2)));
+      Parameters.Add(new ConstrainedValueParameter<INCAInitializer>("Initialization", "Which method should be used to initialize the matrix. Typically LDA (linear discriminant analysis) should provide a good estimate."));
+      Parameters.Add(new FixedValueParameter<IntValue>("NeighborSamples", "How many of the neighbors should be sampled in order to speed up the calculation. This should be at least the value of k and at most the number of training instances minus one.", new IntValue(50)));
+      Parameters.Add(new FixedValueParameter<IntValue>("Iterations", "How many iterations the conjugate gradient (CG) method should be allowed to perform. The method might still terminate earlier if a local optima has already been reached.", new IntValue(20)));
+
+      INCAInitializer defaultInitializer = null;
+      foreach (var initializer in ApplicationManager.Manager.GetInstances<INCAInitializer>().OrderBy(x => x.ItemName)) {
+        if (initializer is LDAInitializer) defaultInitializer = initializer;
+        InitializationParameter.ValidValues.Add(initializer);
+      }
+      if (defaultInitializer != null) InitializationParameter.Value = defaultInitializer;
+
+      Problem = new ClassificationProblem();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NcaAlgorithm(this, cloner);
+    }
+
+    public override void Prepare() {
+      if (Problem != null) base.Prepare();
+    }
+
+    protected override void Run() {
+      var initializer = InitializationParameter.Value;
+
+      var clonedProblem = (IClassificationProblemData)Problem.ProblemData.Clone();
+      var model = Train(clonedProblem, K, Dimensions, NeighborSamples, Iterations, initializer.Initialize(clonedProblem, Dimensions), ReportQuality, CancellationToken.None);
+      Results.Add(new Result("ClassificationSolution", "The classification solution.", model.CreateClassificationSolution(clonedProblem)));
+    }
+
+    public static INcaClassificationSolution CreateClassificationSolution(IClassificationProblemData data, int k, int dimensions, int neighborSamples, int iterations, INCAInitializer initializer) {
+      var clonedProblem = (IClassificationProblemData)data.Clone();
+      var model = Train(clonedProblem, k, dimensions, neighborSamples, iterations, initializer);
+      return model.CreateClassificationSolution(clonedProblem);
+    }
+
+    public static INcaModel Train(IClassificationProblemData problemData, int k, int dimensions, int neighborSamples, int iterations, INCAInitializer initializer) {
+      return Train(problemData, k, dimensions, neighborSamples, iterations, initializer.Initialize(problemData, dimensions), null, CancellationToken.None);
+    }
+
+    public static INcaModel Train(IClassificationProblemData problemData, int k, int neighborSamples, int iterations, double[,] initalMatrix) {
+      var matrix = new double[initalMatrix.Length];
+      for (int i = 0; i < initalMatrix.GetLength(0); i++)
+        for (int j = 0; j < initalMatrix.GetLength(1); j++)
+          matrix[i * initalMatrix.GetLength(1) + j] = initalMatrix[i, j];
+      return Train(problemData, k, initalMatrix.GetLength(1), neighborSamples, iterations, matrix, null, CancellationToken.None);
+    }
+
+    private static INcaModel Train(IClassificationProblemData data, int k, int dimensions, int neighborSamples, int iterations, double[] matrix, Reporter reporter, CancellationToken cancellation) {
+      var scaling = new Scaling(data.Dataset, data.AllowedInputVariables, data.TrainingIndices);
+      var scaledData = AlglibUtil.PrepareAndScaleInputMatrix(data.Dataset, data.AllowedInputVariables, data.TrainingIndices, scaling);
+      var classes = data.Dataset.GetDoubleValues(data.TargetVariable, data.TrainingIndices).ToArray();
+      var attributes = scaledData.GetLength(1);
+
+      alglib.mincgstate state;
+      alglib.mincgreport rep;
+      alglib.mincgcreate(matrix, out state);
+      alglib.mincgsetcond(state, 0, 0, 0, iterations);
+      alglib.mincgsetxrep(state, true);
+      int neighborSampleSize = neighborSamples;
+      Optimize(state, scaledData, classes, dimensions, neighborSampleSize, cancellation, reporter);
+      alglib.mincgresults(state, out matrix, out rep);
+
+      var transformationMatrix = new double[attributes, dimensions];
+      var counter = 0;
+      for (var i = 0; i < attributes; i++)
+        for (var j = 0; j < dimensions; j++)
+          transformationMatrix[i, j] = matrix[counter++];
+
+      return new NcaModel(k, transformationMatrix, data.Dataset, data.TrainingIndices, data.TargetVariable, data.AllowedInputVariables, scaling, data.ClassValues.ToArray());
+    }
+
+    private static void Optimize(alglib.mincgstate state, double[,] data, double[] classes, int dimensions, int neighborSampleSize, CancellationToken cancellation, Reporter reporter) {
+      while (alglib.mincgiteration(state)) {
+        if (cancellation.IsCancellationRequested) break;
+        if (state.needfg) {
+          Gradient(state.x, ref state.innerobj.f, state.innerobj.g, data, classes, dimensions, neighborSampleSize);
+          continue;
+        }
+        if (state.innerobj.xupdated) {
+          if (reporter != null)
+            reporter(state.innerobj.f, state.innerobj.x);
+          continue;
+        }
+        throw new InvalidOperationException("Neighborhood Components Analysis: Error in Optimize() (some derivatives were not provided?)");
+      }
+    }
+
+    private static void Gradient(double[] A, ref double func, double[] grad, double[,] data, double[] classes, int dimensions, int neighborSampleSize) {
+      var instances = data.GetLength(0);
+      var attributes = data.GetLength(1);
+
+      var AMatrix = new Matrix(A, A.Length / dimensions, dimensions);
+
+      alglib.sparsematrix probabilities;
+      alglib.sparsecreate(instances, instances, out probabilities);
+      var transformedDistances = new Dictionary<int, double>(instances);
+      for (int i = 0; i < instances; i++) {
+        var iVector = new Matrix(GetRow(data, i), data.GetLength(1));
+        for (int k = 0; k < instances; k++) {
+          if (k == i) {
+            transformedDistances.Remove(k);
+            continue;
+          }
+          var kVector = new Matrix(GetRow(data, k));
+          transformedDistances[k] = Math.Exp(-iVector.Multiply(AMatrix).Subtract(kVector.Multiply(AMatrix)).SquaredVectorLength());
+        }
+        var sample = transformedDistances.OrderByDescending(x => x.Value).Take(neighborSampleSize).ToArray();
+        var normalization = sample.Sum(x => x.Value);
+        if (normalization > 0) {
+          foreach (var s in sample) {
+            if (s.Value <= 0) break;
+            alglib.sparseset(probabilities, i, s.Key, s.Value / normalization);
+          }
+        }
+      }
+      alglib.sparseconverttocrs(probabilities); // needed to enumerate in order (top-down and left-right)
+
+      int t0 = 0, t1 = 0, r, c;
+      double val;
+      var pi = new double[instances];
+      while (alglib.sparseenumerate(probabilities, ref t0, ref t1, out r, out c, out val)) {
+        if (classes[r].IsAlmost(classes[c])) {
+          pi[r] += val;
+        }
+      }
+
+      var innerSum = new double[attributes, attributes];
+      while (alglib.sparseenumerate(probabilities, ref t0, ref t1, out r, out c, out val)) {
+        var vector = new Matrix(GetRow(data, r)).Subtract(new Matrix(GetRow(data, c)));
+        vector.OuterProduct(vector).Multiply(val * pi[r]).AddTo(innerSum);
+
+        if (classes[r].IsAlmost(classes[c])) {
+          vector.OuterProduct(vector).Multiply(-val).AddTo(innerSum);
+        }
+      }
+
+      func = -pi.Sum();
+
+      r = 0;
+      var newGrad = AMatrix.Multiply(-2.0).Transpose().Multiply(new Matrix(innerSum)).Transpose();
+      foreach (var g in newGrad) {
+        grad[r++] = g;
+      }
+    }
+
+    private void ReportQuality(double func, double[] coefficients) {
+      var instances = Problem.ProblemData.TrainingIndices.Count();
+      DataTable qualities;
+      if (!Results.ContainsKey("Optimization")) {
+        qualities = new DataTable("Optimization");
+        qualities.Rows.Add(new DataRow("Quality", string.Empty));
+        Results.Add(new Result("Optimization", qualities));
+      } else qualities = (DataTable)Results["Optimization"].Value;
+      qualities.Rows["Quality"].Values.Add(-func / instances);
+
+      if (!Results.ContainsKey("Quality")) {
+        Results.Add(new Result("Quality", new DoubleValue(-func / instances)));
+      } else ((DoubleValue)Results["Quality"].Value).Value = -func / instances;
+    }
+
+    private static IEnumerable<double> GetRow(double[,] data, int row) {
+      for (int i = 0; i < data.GetLength(1); i++)
+        yield return data[row, i];
+    }
+  }
+}
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaClassificationSolution.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaClassificationSolution.cs	(revision 8477)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaClassificationSolution.cs	(revision 8477)
@@ -0,0 +1,55 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("NCA Classification Solution", "")]
+  [StorableClass]
+  public class NcaClassificationSolution : ClassificationSolution, INcaClassificationSolution {
+
+    public new INcaModel Model {
+      get { return (INcaModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    protected NcaClassificationSolution(bool deserializing) : base(deserializing) { }
+    protected NcaClassificationSolution(NcaClassificationSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NcaClassificationSolution(IClassificationProblemData problemData, INcaModel ncaModel)
+      : base(ncaModel, problemData) {
+      RecalculateResults();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NcaClassificationSolution(this, cloner);
+    }
+
+    protected override void RecalculateResults() {
+      CalculateResults();
+    }
+  }
+}
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs	(revision 8477)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs	(revision 8477)
@@ -0,0 +1,113 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("NCA Model", "")]
+  [StorableClass]
+  public class NcaModel : NamedItem, INcaModel {
+
+    [Storable]
+    private Scaling scaling;
+    [Storable]
+    private double[,] transformationMatrix;
+    public double[,] TransformationMatrix {
+      get { return (double[,])transformationMatrix.Clone(); }
+    }
+    [Storable]
+    private string[] allowedInputVariables;
+    [Storable]
+    private string targetVariable;
+    [Storable]
+    private INearestNeighbourModel nnModel;
+    [Storable]
+    private double[] classValues;
+
+    [StorableConstructor]
+    protected NcaModel(bool deserializing) : base(deserializing) { }
+    protected NcaModel(NcaModel original, Cloner cloner)
+      : base(original, cloner) {
+      this.scaling = cloner.Clone(original.scaling);
+      this.transformationMatrix = (double[,])original.transformationMatrix.Clone();
+      this.allowedInputVariables = (string[])original.allowedInputVariables.Clone();
+      this.targetVariable = original.targetVariable;
+      this.nnModel = cloner.Clone(original.nnModel);
+      this.classValues = (double[])original.classValues.Clone();
+    }
+    public NcaModel(int k, double[,] transformationMatrix, Dataset dataset, IEnumerable<int> rows, string targetVariable, IEnumerable<string> allowedInputVariables, Scaling scaling, double[] classValues) {
+      Name = ItemName;
+      Description = ItemDescription;
+      this.scaling = scaling;
+      this.transformationMatrix = (double[,])transformationMatrix.Clone();
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+      this.targetVariable = targetVariable;
+      this.classValues = (double[])classValues.Clone();
+
+      var ds = ReduceDataset(dataset, rows);
+      nnModel = new NearestNeighbourModel(ds, Enumerable.Range(0, ds.Rows), k, ds.VariableNames.Last(), ds.VariableNames.Take(transformationMatrix.GetLength(1)), classValues);
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NcaModel(this, cloner);
+    }
+
+    public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
+      var ds = ReduceDataset(dataset, rows);
+      return nnModel.GetEstimatedClassValues(ds, Enumerable.Range(0, ds.Rows));
+    }
+
+    public INcaClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new NcaClassificationSolution(problemData, this);
+    }
+
+    IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {
+      return CreateClassificationSolution(problemData);
+    }
+
+    public double[,] Reduce(Dataset dataset, IEnumerable<int> rows) {
+      var scaledData = AlglibUtil.PrepareAndScaleInputMatrix(dataset, allowedInputVariables, rows, scaling);
+      var targets = dataset.GetDoubleValues(targetVariable, rows).ToArray();
+      var result = new double[scaledData.GetLength(0), transformationMatrix.GetLength(1) + 1];
+      for (int i = 0; i < scaledData.GetLength(0); i++)
+        for (int j = 0; j < scaledData.GetLength(1); j++) {
+          for (int x = 0; x < transformationMatrix.GetLength(1); x++) {
+            result[i, x] += scaledData[i, j] * transformationMatrix[j, x];
+          }
+          result[i, transformationMatrix.GetLength(1)] = targets[i];
+        }
+      return result;
+    }
+
+    public Dataset ReduceDataset(Dataset dataset, IEnumerable<int> rows) {
+      return new Dataset(Enumerable
+          .Range(0, transformationMatrix.GetLength(1))
+          .Select(x => "X" + x.ToString())
+          .Concat(targetVariable.ToEnumerable()),
+        Reduce(dataset, rows));
+    }
+  }
+}
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs	(revision 8477)
@@ -21,16 +21,12 @@
 
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
 using HeuristicLab.Data;
-using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
 using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
-using HeuristicLab.Problems.DataAnalysis.Symbolic;
-using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
-using HeuristicLab.Parameters;
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
@@ -84,29 +80,15 @@
 
     public static IClassificationSolution CreateNearestNeighbourClassificationSolution(IClassificationProblemData problemData, int k) {
-      Dataset dataset = problemData.Dataset;
-      string targetVariable = problemData.TargetVariable;
-      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
-      IEnumerable<int> rows = problemData.TrainingIndices;
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
-      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
-        throw new NotSupportedException("Nearest neighbour classification does not support NaN or infinity values in the input dataset.");
+      var problemDataClone = (IClassificationProblemData)problemData.Clone();
+      return new NearestNeighbourClassificationSolution(problemDataClone, Train(problemDataClone, k));
+    }
 
-      alglib.nearestneighbor.kdtree kdtree = new alglib.nearestneighbor.kdtree();
-
-      int nRows = inputMatrix.GetLength(0);
-      int nFeatures = inputMatrix.GetLength(1) - 1;
-      double[] classValues = dataset.GetDoubleValues(targetVariable).Distinct().OrderBy(x => x).ToArray();
-      int nClasses = classValues.Count();
-      // map original class values to values [0..nClasses-1]
-      Dictionary<double, double> classIndices = new Dictionary<double, double>();
-      for (int i = 0; i < nClasses; i++) {
-        classIndices[classValues[i]] = i;
-      }
-      for (int row = 0; row < nRows; row++) {
-        inputMatrix[row, nFeatures] = classIndices[inputMatrix[row, nFeatures]];
-      }
-      alglib.nearestneighbor.kdtreebuild(inputMatrix, nRows, inputMatrix.GetLength(1) - 1, 1, 2, kdtree);
-      var problemDataClone = (IClassificationProblemData) problemData.Clone();
-      return new NearestNeighbourClassificationSolution(problemDataClone, new NearestNeighbourModel(kdtree, k, targetVariable, allowedInputVariables, problemDataClone.ClassValues.ToArray()));
+    public static INearestNeighbourModel Train(IClassificationProblemData problemData, int k) {
+      return new NearestNeighbourModel(problemData.Dataset,
+        problemData.TrainingIndices,
+        k,
+        problemData.TargetVariable,
+        problemData.AllowedInputVariables,
+        problemData.ClassValues.ToArray());
     }
     #endregion
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs	(revision 8477)
@@ -33,5 +33,5 @@
   /// </summary>
   [StorableClass]
-  [Item("NearestNeighbourModel", "Represents a neural network for regression and classification.")]
+  [Item("NearestNeighbourModel", "Represents a nearest neighbour model for regression and classification.")]
   public sealed class NearestNeighbourModel : NamedItem, INearestNeighbourModel {
 
@@ -56,4 +56,5 @@
     [Storable]
     private int k;
+
     [StorableConstructor]
     private NearestNeighbourModel(bool deserializing)
@@ -95,14 +96,37 @@
         this.classValues = (double[])original.classValues.Clone();
     }
-    public NearestNeighbourModel(alglib.nearestneighbor.kdtree kdTree, int k, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues = null)
-      : base() {
-      this.name = ItemName;
-      this.description = ItemDescription;
-      this.kdTree = kdTree;
+    public NearestNeighbourModel(Dataset dataset, IEnumerable<int> rows, int k, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues = null) {
+      Name = ItemName;
+      Description = ItemDescription;
       this.k = k;
       this.targetVariable = targetVariable;
       this.allowedInputVariables = allowedInputVariables.ToArray();
-      if (classValues != null)
+
+      var inputMatrix = AlglibUtil.PrepareInputMatrix(dataset,
+                                   allowedInputVariables.Concat(new string[] { targetVariable }),
+                                   rows);
+
+      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
+        throw new NotSupportedException(
+          "Nearest neighbour classification does not support NaN or infinity values in the input dataset.");
+
+      this.kdTree = new alglib.nearestneighbor.kdtree();
+
+      var nRows = inputMatrix.GetLength(0);
+      var nFeatures = inputMatrix.GetLength(1) - 1;
+
+      if (classValues != null) {
         this.classValues = (double[])classValues.Clone();
+        int nClasses = classValues.Length;
+        // map original class values to values [0..nClasses-1]
+        var classIndices = new Dictionary<double, double>();
+        for (int i = 0; i < nClasses; i++)
+          classIndices[classValues[i]] = i;
+
+        for (int row = 0; row < nRows; row++) {
+          inputMatrix[row, nFeatures] = classIndices[inputMatrix[row, nFeatures]];
+        }
+      }
+      alglib.nearestneighbor.kdtreebuild(inputMatrix, nRows, inputMatrix.GetLength(1) - 1, 1, 2, kdTree);
     }
 
@@ -140,4 +164,5 @@
 
     public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
+      if (classValues == null) throw new InvalidOperationException("No class values are defined.");
       double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
 
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs	(revision 8477)
@@ -21,16 +21,11 @@
 
 using System;
-using System.Collections.Generic;
-using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
 using HeuristicLab.Data;
-using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
 using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
-using HeuristicLab.Problems.DataAnalysis.Symbolic;
-using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
-using HeuristicLab.Parameters;
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
@@ -84,19 +79,14 @@
 
     public static IRegressionSolution CreateNearestNeighbourRegressionSolution(IRegressionProblemData problemData, int k) {
-      Dataset dataset = problemData.Dataset;
-      string targetVariable = problemData.TargetVariable;
-      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
-      IEnumerable<int> rows = problemData.TrainingIndices;
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
-      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
-        throw new NotSupportedException("Nearest neighbour regression does not support NaN or infinity values in the input dataset.");
+      var clonedProblemData = (IRegressionProblemData)problemData.Clone();
+      return new NearestNeighbourRegressionSolution(clonedProblemData, Train(problemData, k));
+    }
 
-      alglib.nearestneighbor.kdtree kdtree = new alglib.nearestneighbor.kdtree();
-
-      int nRows = inputMatrix.GetLength(0);
-
-      alglib.nearestneighbor.kdtreebuild(inputMatrix, nRows, inputMatrix.GetLength(1) - 1, 1, 2, kdtree);
-
-      return new NearestNeighbourRegressionSolution((IRegressionProblemData)problemData.Clone(), new NearestNeighbourModel(kdtree, k, targetVariable, allowedInputVariables));
+    public static INearestNeighbourModel Train(IRegressionProblemData problemData, int k) {
+      return new NearestNeighbourModel(problemData.Dataset,
+        problemData.TrainingIndices,
+        k,
+        problemData.TargetVariable,
+        problemData.AllowedInputVariables);
     }
     #endregion
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame	(revision 8477)
@@ -28,5 +28,5 @@
   [Plugin("HeuristicLab.Algorithms.DataAnalysis", "Provides wrappers for data analysis algorithms implemented in external libraries (linear regression, linear discriminant analysis, k-means clustering, support vector classification and regression)", "3.4.3.$WCREV$")]
   [PluginFile("HeuristicLab.Algorithms.DataAnalysis-3.4.dll", PluginFileType.Assembly)]
-  [PluginDependency("HeuristicLab.ALGLIB", "3.5.0")]
+  [PluginDependency("HeuristicLab.ALGLIB", "3.6.0")]
   [PluginDependency("HeuristicLab.Algorithms.GradientDescent", "3.3")]
   [PluginDependency("HeuristicLab.Analysis", "3.3")]
@@ -47,4 +47,5 @@
   [PluginDependency("HeuristicLab.Problems.DataAnalysis.Symbolic.Regression", "3.4")]
   [PluginDependency("HeuristicLab.LibSVM", "1.6.3")]
+  [PluginDependency("HeuristicLab.Random", "3.3")]
   public class HeuristicLabAlgorithmsDataAnalysisPlugin : PluginBase {
   }
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Encodings.SymbolicExpressionTreeEncoding/3.4/SymbolicExpressionGrammar.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Encodings.SymbolicExpressionTreeEncoding/3.4/SymbolicExpressionGrammar.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Encodings.SymbolicExpressionTreeEncoding/3.4/SymbolicExpressionGrammar.cs	(revision 8477)
@@ -181,6 +181,5 @@
 
     #region IStatefulItem methods
-    void IStatefulItem.InitializeState()
-    {
+    void IStatefulItem.InitializeState() {
       ReadOnly = false;
     }
@@ -245,5 +244,5 @@
         var groupSymbol = s as GroupSymbol;
         if (groupSymbol != null) RegisterGroupSymbolEvents(groupSymbol);
-        else symbol.Changed += new EventHandler(Symbol_Changed);
+        else s.Changed += new EventHandler(Symbol_Changed);
       }
     }
@@ -255,5 +254,5 @@
         var groupSymbol = s as GroupSymbol;
         if (groupSymbol != null) DeregisterGroupSymbolEvents(groupSymbol);
-        else symbol.Changed -= new EventHandler(Symbol_Changed);
+        else s.Changed -= new EventHandler(Symbol_Changed);
       }
     }
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Encodings.SymbolicExpressionTreeEncoding/3.4/SymbolicExpressionTreeTerminalNode.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Encodings.SymbolicExpressionTreeEncoding/3.4/SymbolicExpressionTreeTerminalNode.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Encodings.SymbolicExpressionTreeEncoding/3.4/SymbolicExpressionTreeTerminalNode.cs	(revision 8477)
@@ -30,7 +30,5 @@
   public abstract class SymbolicExpressionTreeTerminalNode : SymbolicExpressionTreeNode {
     public override IEnumerable<ISymbolicExpressionTreeNode> Subtrees {
-      get {
-        return Enumerable.Empty<ISymbolicExpressionTreeNode>();
-      }
+      get { return Enumerable.Empty<ISymbolicExpressionTreeNode>(); }
     }
 
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic.TimeSeriesPrognosis/3.4/SingleObjective/SymbolicTimeSeriesPrognosisSingleObjectiveProblem.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic.TimeSeriesPrognosis/3.4/SingleObjective/SymbolicTimeSeriesPrognosisSingleObjectiveProblem.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic.TimeSeriesPrognosis/3.4/SingleObjective/SymbolicTimeSeriesPrognosisSingleObjectiveProblem.cs	(revision 8477)
@@ -75,4 +75,11 @@
       var grammar = SymbolicExpressionTreeGrammar as TypeCoherentExpressionGrammar;
       if (grammar != null) grammar.ConfigureAsDefaultTimeSeriesPrognosisGrammar();
+      UpdateGrammar();
+    }
+    protected override void UpdateGrammar() {
+      base.UpdateGrammar();
+      foreach (var autoregressiveSymbol in SymbolicExpressionTreeGrammar.Symbols.OfType<AutoregressiveTargetVariable>()) {
+        if (!autoregressiveSymbol.Fixed) autoregressiveSymbol.VariableNames = ProblemData.TargetVariable.ToEnumerable();
+      }
     }
 
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic.Views/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic.Views-3.4.csproj
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic.Views/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic.Views-3.4.csproj	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic.Views/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic.Views-3.4.csproj	(revision 8477)
@@ -93,6 +93,8 @@
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="ALGLIB-3.5.0, Version=3.5.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
-    <Reference Include="HeuristicLab.ALGLIB-3.5.0, Version=3.5.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
+    <Reference Include="ALGLIB-3.6.0, Version=3.6.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\..\..\trunk\sources\bin\ALGLIB-3.6.0.dll</HintPath>
+      <Private>False</Private>
+    </Reference>
     <Reference Include="HeuristicLab.Collections-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
     <Reference Include="HeuristicLab.Common-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
@@ -104,5 +106,4 @@
     <Reference Include="HeuristicLab.MainForm-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
     <Reference Include="HeuristicLab.MainForm.WindowsForms-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
-    <Reference Include="HeuristicLab.MathJax-1.1, Version=1.1.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
     <Reference Include="HeuristicLab.Optimization-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
     <Reference Include="HeuristicLab.PluginInfrastructure-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic.Views/3.4/Plugin.cs.frame
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic.Views/3.4/Plugin.cs.frame	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic.Views/3.4/Plugin.cs.frame	(revision 8477)
@@ -29,5 +29,5 @@
   [PluginFile("HeuristicLab.Problems.DataAnalysis.Symbolic.Views-3.4.dll", PluginFileType.Assembly)]
   [PluginFile("displayModelFrame.html", PluginFileType.Data)]
-  [PluginDependency("HeuristicLab.ALGLIB", "3.5")]
+  [PluginDependency("HeuristicLab.ALGLIB", "3.6.0")]
   [PluginDependency("HeuristicLab.Collections", "3.3")]
   [PluginDependency("HeuristicLab.Common", "3.3")]
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic.Views/3.4/Symbols/VariableView.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic.Views/3.4/Symbols/VariableView.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic.Views/3.4/Symbols/VariableView.cs	(revision 8477)
@@ -92,4 +92,5 @@
     protected override void SetEnabledStateOfControls() {
       base.SetEnabledStateOfControls();
+      enabledCheckBox.Enabled = Content != null && Content.VariableNames.Any() && !Locked && !ReadOnly;
       weightInitializationMuTextBox.Enabled = Content != null;
       weightInitializationMuTextBox.ReadOnly = ReadOnly;
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Grammars/FullFunctionalExpressionGrammar.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Grammars/FullFunctionalExpressionGrammar.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Grammars/FullFunctionalExpressionGrammar.cs	(revision 8477)
@@ -116,8 +116,11 @@
       var laggedVariable = new LaggedVariable();
       laggedVariable.InitialFrequency = 0.0;
+      var autoregressiveVariable = new AutoregressiveTargetVariable();
+      autoregressiveVariable.InitialFrequency = 0.0;
+      autoregressiveVariable.Enabled = false;
 
       var allSymbols = new List<Symbol>() { add, sub, mul, div, mean, sin, cos, tan, log, square, pow, sqrt, root, exp,
         airyA, airyB, bessel, cosineIntegral, dawson, erf, expIntegralEi, fresnelCosineIntegral, fresnelSineIntegral, gamma, hypCosineIntegral, hypSineIntegral, norm, psi, sineIntegral,
-        @if, gt, lt, and, or, not, timeLag, integral, derivative, constant, variableSymbol, laggedVariable, variableCondition };
+        @if, gt, lt, and, or, not, timeLag, integral, derivative, constant, variableSymbol, laggedVariable,autoregressiveVariable, variableCondition };
       var unaryFunctionSymbols = new List<Symbol>() { square, sqrt, sin, cos, tan, log, exp, not, timeLag, integral, derivative,
         airyA, airyB, bessel, cosineIntegral, dawson, erf, expIntegralEi, fresnelCosineIntegral, fresnelSineIntegral, gamma, hypCosineIntegral, hypSineIntegral, norm, psi, sineIntegral
@@ -126,5 +129,5 @@
       var binaryFunctionSymbols = new List<Symbol>() { pow, root, gt, lt, variableCondition };
       var ternarySymbols = new List<Symbol>() { add, sub, mul, div, mean, and, or };
-      var terminalSymbols = new List<Symbol>() { variableSymbol, constant, laggedVariable };
+      var terminalSymbols = new List<Symbol>() { variableSymbol, constant, laggedVariable, autoregressiveVariable };
 
       foreach (var symb in allSymbols)
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Grammars/TypeCoherentExpressionGrammar.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Grammars/TypeCoherentExpressionGrammar.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Grammars/TypeCoherentExpressionGrammar.cs	(revision 8477)
@@ -104,4 +104,5 @@
       var variableSymbol = new Variable();
       var laggedVariable = new LaggedVariable();
+      var autoregressiveVariable = new AutoregressiveTargetVariable();
       #endregion
 
@@ -122,5 +123,5 @@
       var conditionalSymbols = new GroupSymbol(ConditionalSymbolsName, new List<ISymbol> { conditionSymbols, comparisonSymbols, booleanOperationSymbols });
 
-      var timeSeriesSymbols = new GroupSymbol(TimeSeriesSymbolsName, new List<ISymbol> { timeLag, integral, derivative, laggedVariable });
+      var timeSeriesSymbols = new GroupSymbol(TimeSeriesSymbolsName, new List<ISymbol> { timeLag, integral, derivative, laggedVariable, autoregressiveVariable });
       #endregion
 
@@ -152,4 +153,5 @@
       SetSubtreeCount(derivative, 1, 1);
       SetSubtreeCount(laggedVariable, 0, 0);
+      SetSubtreeCount(autoregressiveVariable, 0, 0);
       #endregion
 
@@ -227,9 +229,14 @@
 
     public void ConfigureAsDefaultTimeSeriesPrognosisGrammar() {
-      Symbols.First(s => s is Variable).Enabled = false;
+      Symbols.First(s => s is Average).Enabled = false;
       Symbols.First(s => s.Name == TrigonometricFunctionsName).Enabled = false;
       Symbols.First(s => s.Name == PowerFunctionsName).Enabled = false;
       Symbols.First(s => s.Name == ConditionalSymbolsName).Enabled = false;
       Symbols.First(s => s.Name == SpecialFunctionsName).Enabled = false;
+
+      Symbols.First(s => s.Name == TimeSeriesSymbolsName).Enabled = true;
+      Symbols.First(s => s is Derivative).Enabled = false;
+      Symbols.First(s => s is Integral).Enabled = false;
+      Symbols.First(s => s is TimeLag).Enabled = false;
     }
   }
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj	(revision 8477)
@@ -186,4 +186,5 @@
     <Compile Include="Symbols\AiryB.cs" />
     <Compile Include="Symbols\Bessel.cs" />
+    <Compile Include="Symbols\AutoregressiveVariable.cs" />
     <Compile Include="Symbols\Erf.cs" />
     <Compile Include="Symbols\Norm.cs" />
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/OpCodes.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/OpCodes.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/OpCodes.cs	(revision 8477)
@@ -103,4 +103,5 @@
       { typeof(Variable), OpCodes.Variable },
       { typeof(LaggedVariable), OpCodes.LagVariable },
+      { typeof(AutoregressiveTargetVariable),OpCodes.LagVariable},
       { typeof(Constant), OpCodes.Constant },
       { typeof(Argument), OpCodes.Arg },
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Symbols/AutoregressiveVariable.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Symbols/AutoregressiveVariable.cs	(revision 8477)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Symbols/AutoregressiveVariable.cs	(revision 8477)
@@ -0,0 +1,39 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+namespace HeuristicLab.Problems.DataAnalysis.Symbolic {
+  [StorableClass]
+  [Item("LaggedVariable", "Represents a variable value with a time offset.")]
+  public sealed class AutoregressiveTargetVariable : LaggedVariable {
+    [StorableConstructor]
+    private AutoregressiveTargetVariable(bool deserializing) : base(deserializing) { }
+    private AutoregressiveTargetVariable(AutoregressiveTargetVariable original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public AutoregressiveTargetVariable() : base("Autoregressive Target Variable", "Represents the target variable with a time offset.") { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new AutoregressiveTargetVariable(this, cloner);
+    }
+  }
+}
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Symbols/LaggedVariable.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Symbols/LaggedVariable.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Symbols/LaggedVariable.cs	(revision 8477)
@@ -27,5 +27,5 @@
   [StorableClass]
   [Item("LaggedVariable", "Represents a variable value with a time offset.")]
-  public sealed class LaggedVariable : Variable {
+  public class LaggedVariable : Variable {
     [Storable]
     private int minLag;
@@ -41,13 +41,15 @@
     }
     [StorableConstructor]
-    private LaggedVariable(bool deserializing) : base(deserializing) { }
-    private LaggedVariable(LaggedVariable original, Cloner cloner)
+    protected LaggedVariable(bool deserializing) : base(deserializing) { }
+    protected LaggedVariable(LaggedVariable original, Cloner cloner)
       : base(original, cloner) {
       minLag = original.minLag;
       maxLag = original.maxLag;
     }
-    public LaggedVariable()
-      : base("LaggedVariable", "Represents a variable value with a time offset.") {
-      minLag = -1; maxLag = -1;
+    public LaggedVariable() : this("LaggedVariable", "Represents a variable value with a time offset.") { }
+    protected LaggedVariable(string name, string description)
+      : base(name, description) {
+      MinLag = -1;
+      MaxLag = -1;
     }
 
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Symbols/LaggedVariableTreeNode.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Symbols/LaggedVariableTreeNode.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Symbols/LaggedVariableTreeNode.cs	(revision 8477)
@@ -37,4 +37,8 @@
     }
 
+    public override bool HasLocalParameters {
+      get { return true; }
+    }
+
     [StorableConstructor]
     private LaggedVariableTreeNode(bool deserializing) : base(deserializing) { }
@@ -43,13 +47,7 @@
       lag = original.lag;
     }
-    private LaggedVariableTreeNode() { }
 
     public LaggedVariableTreeNode(LaggedVariable variableSymbol) : base(variableSymbol) { }
 
-    public override bool HasLocalParameters {
-      get {
-        return true;
-      }
-    }
 
     public override void ResetLocalParameters(IRandom random) {
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Symbols/Variable.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Symbols/Variable.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Symbols/Variable.cs	(revision 8477)
@@ -101,4 +101,15 @@
     }
 
+    public override bool Enabled {
+      get {
+        if (variableNames.Count == 0) return false;
+        return base.Enabled;
+      }
+      set {
+        if (variableNames.Count == 0) base.Enabled = false;
+        else base.Enabled = value;
+      }
+    }
+
     private const int minimumArity = 0;
     private const int maximumArity = 0;
Index: /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/TimeSeriesPrognosis/TimeSeriesPrognosisProblemData.cs
===================================================================
--- /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/TimeSeriesPrognosis/TimeSeriesPrognosisProblemData.cs	(revision 8476)
+++ /branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/TimeSeriesPrognosis/TimeSeriesPrognosisProblemData.cs	(revision 8477)
@@ -22,4 +22,5 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -1542,5 +1543,5 @@
       defaultDataset = new Dataset(new string[] { "x" }, mackey_glass_17);
       defaultDataset.Name = "Mackey-Glass (t=17) Time Series Benchmark Dataset";
-      defaultAllowedInputVariables = new List<string>() { "x" };
+      defaultAllowedInputVariables = Enumerable.Empty<string>();
       defaultTargetVariable = "x";
     }
