Changeset 8742 for branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs

-                      r7738
+                      r8742
       executionState = ExecutionState.Stopped;
       runs = new RunCollection();
+      runs = new RunCollection { AlgorithmName = name };
       runsCounter = 0;
 …
     #endregion
+    protected override void OnNameChanged() {
+      base.OnNameChanged();
+      Runs.AlgorithmName = Name;
+    }
     public void Prepare() {
       if (ExecutionState == ExecutionState.Started)
 …
         problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
         // clone models
+        var ensembleSolution = new ClassificationEnsembleSolution(
+          solutions.Value.Select(x => cloner.Clone(x.Model)),
+          problemDataClone,
+          solutions.Value.Select(x => cloner.Clone(x.ProblemData.TrainingPartition)),
+          solutions.Value.Select(x => cloner.Clone(x.ProblemData.TestPartition)));
+        var ensembleSolution = new ClassificationEnsembleSolution(problemDataClone);
+        ensembleSolution.AddClassificationSolutions(solutions.Value);
         aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceConst.cs

-                      r8477
+                      r8742
 using System;
+using System.Collections.Generic;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Data;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 …
   [Item(Name = "CovarianceConst",
     Description = "Constant covariance function for Gaussian processes.")]
+  public class CovarianceConst : Item, ICovarianceFunction {
+  public sealed class CovarianceConst : ParameterizedNamedItem, ICovarianceFunction {
     [Storable]
+    private double sf2;
+    public double Scale { get { return sf2; } }
+    private double scale;
+    [Storable]
+    private readonly HyperParameter<DoubleValue> scaleParameter;
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return scaleParameter; }
+    }
     [StorableConstructor]
     protected CovarianceConst(bool deserializing)
+    private CovarianceConst(bool deserializing)
       : base(deserializing) {
+    }
     protected CovarianceConst(CovarianceConst original, Cloner cloner)
+    private CovarianceConst(CovarianceConst original, Cloner cloner)
       : base(original, cloner) {
+      this.sf2 = original.sf2;
+      this.scaleParameter = cloner.Clone(original.scaleParameter);
+      this.scale = original.scale;
+      RegisterEvents();
+    }
     public CovarianceConst()
       : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+      scaleParameter = new HyperParameter<DoubleValue>("Scale", "The scale of the constant covariance function.");
+      Parameters.Add(scaleParameter);
+      RegisterEvents();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      RegisterEvents();
+    }
+    // caching
+    private void RegisterEvents() {
+      Util.AttachValueChangeHandler<DoubleValue, double>(scaleParameter, () => { scale = scaleParameter.Value.Value; });
+    }
     public override IDeepCloneable Clone(Cloner cloner) {
 …
     public int GetNumberOfParameters(int numberOfVariables) {
       return 1;
+      return scaleParameter.Fixed ? 0 : 1;
+    }
     public void SetParameter(double[] hyp) {
+      this.sf2 = Math.Exp(2 * hyp[0]);
+    }
+    public void SetData(double[,] x) {
+      // nothing to do
+      if (!scaleParameter.Fixed && hyp.Length == 1) {
+        scaleParameter.SetValue(new DoubleValue(Math.Exp(2 * hyp[0])));
+      } else {
+        throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceConst", "hyp");
+      }
+    }
+    public void SetData(double[,] x, double[,] xt) {
+      // nothing to do
+    public double GetCovariance(double[,] x, int i, int j, IEnumerable<int> columnIndices) {
+      return scale;
+    }
     public double GetCovariance(int i, int j) {
       return sf2;
+    public IEnumerable<double> GetGradient(double[,] x, int i, int j, IEnumerable<int> columnIndices) {
+      yield return 2.0 * scale;
+    }
+    public double GetGradient(int i, int j, int k) {
+      if (k != 0) throw new ArgumentException("CovarianceConst has only one hyperparameters", "k");
+      return 2 * sf2;
+    public double GetCrossCovariance(double[,] x, double[,] xt, int i, int j, IEnumerable<int> columnIndices) {
+      return scale;
+    }
+  }

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceLinear.cs

-                      r8477
+                      r8742
 using System;
+using System.Collections.Generic;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
 …
   [StorableClass]
   [Item(Name = "CovarianceLinear", Description = "Linear covariance function for Gaussian processes.")]
+  public class CovarianceLinear : Item, ICovarianceFunction {
+    [Storable]
+    private double[,] x;
+    [Storable]
+    private double[,] xt;
+    private double[,] k;
+    private bool symmetric;
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return 0;
+    }
+  public sealed class CovarianceLinear : Item, ICovarianceFunction {
     [StorableConstructor]
     protected CovarianceLinear(bool deserializing) : base(deserializing) { }
     protected CovarianceLinear(CovarianceLinear original, Cloner cloner)
+    private CovarianceLinear(bool deserializing) : base(deserializing) { }
+    private CovarianceLinear(CovarianceLinear original, Cloner cloner)
       : base(original, cloner) {
-      if (original.x != null) {
-        this.x = new double[original.x.GetLength(0), original.x.GetLength(1)];
-        Array.Copy(original.x, this.x, x.Length);
-        this.xt = new double[original.xt.GetLength(0), original.xt.GetLength(1)];
-        Array.Copy(original.xt, this.xt, xt.Length);
-        this.k = new double[original.k.GetLength(0), original.k.GetLength(1)];
-        Array.Copy(original.k, this.k, k.Length);
+      }
-      this.symmetric = original.symmetric;
+    }
     public CovarianceLinear()
 …
+    }
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return 0;
+    }
     public void SetParameter(double[] hyp) {
       if (hyp.Length > 0) throw new ArgumentException("No hyperparameters are allowed for the linear covariance function.");
-      k = null;
+    }
+    public void SetData(double[,] x) {
+      SetData(x, x);
+      this.symmetric = true;
+    public double GetCovariance(double[,] x, int i, int j, IEnumerable<int> columnIndices) {
+      return Util.ScalarProd(x, i, j, 1, columnIndices);
+    }
+    public void SetData(double[,] x, double[,] xt) {
+      this.x = x;
+      this.xt = xt;
+      this.symmetric = false;
+      k = null;
+    public IEnumerable<double> GetGradient(double[,] x, int i, int j, IEnumerable<int> columnIndices) {
+      yield break;
+    }
+    public double GetCovariance(int i, int j) {
+      if (k == null) CalculateInnerProduct();
+      return k[i, j];
+    }
+    public double GetGradient(int i, int j, int k) {
+      throw new NotSupportedException("CovarianceLinear does not have hyperparameters.");
+    }
+    private void CalculateInnerProduct() {
+      if (x.GetLength(1) != xt.GetLength(1)) throw new InvalidOperationException();
+      int rows = x.GetLength(0);
+      int cols = xt.GetLength(0);
+      k = new double[rows, cols];
+      if (symmetric) {
+        for (int i = 0; i < rows; i++) {
+          for (int j = i; j < cols; j++) {
+            k[i, j] = Util.ScalarProd(Util.GetRow(x, i),
+                                      Util.GetRow(x, j));
+            k[j, i] = k[i, j];
+          }
+        }
+      } else {
+        for (int i = 0; i < rows; i++) {
+          for (int j = 0; j < cols; j++) {
+            k[i, j] = Util.ScalarProd(Util.GetRow(x, i),
+                                      Util.GetRow(xt, j));
+          }
+        }
+      }
+    public double GetCrossCovariance(double[,] x, double[,] xt, int i, int j, IEnumerable<int> columnIndices) {
+      return Util.ScalarProd(x, i, xt, j);
+    }
+  }

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceNoise.cs

-                      r8477
+                      r8742
 using System;
+using System.Collections.Generic;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Data;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 …
   [Item(Name = "CovarianceNoise",
     Description = "Noise covariance function for Gaussian processes.")]
+  public class CovarianceNoise : Item, ICovarianceFunction {
+  public sealed class CovarianceNoise : ParameterizedNamedItem, ICovarianceFunction {
     [Storable]
     private double sf2;
+    public double Scale { get { return sf2; } }
+    [Storable]
+    private readonly HyperParameter<DoubleValue> scaleParameter;
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return scaleParameter; }
+    }
     [StorableConstructor]
     protected CovarianceNoise(bool deserializing)
+    private CovarianceNoise(bool deserializing)
       : base(deserializing) {
+    }
     protected CovarianceNoise(CovarianceNoise original, Cloner cloner)
+    private CovarianceNoise(CovarianceNoise original, Cloner cloner)
       : base(original, cloner) {
+      this.scaleParameter = cloner.Clone(original.scaleParameter);
       this.sf2 = original.sf2;
+      RegisterEvents();
+    }
     public CovarianceNoise()
       : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+      this.scaleParameter = new HyperParameter<DoubleValue>("Scale", "The scale of noise.");
+      Parameters.Add(this.scaleParameter);
+      RegisterEvents();
+    }
 …
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      RegisterEvents();
+    }
+    private void RegisterEvents() {
+      Util.AttachValueChangeHandler<DoubleValue, double>(scaleParameter, () => { sf2 = scaleParameter.Value.Value; });
+    }
     public int GetNumberOfParameters(int numberOfVariables) {
       return 1;
+      return scaleParameter.Fixed ? 0 : 1;
+    }
     public void SetParameter(double[] hyp) {
+      this.sf2 = Math.Exp(2 * hyp[0]);
+    }
+    public void SetData(double[,] x) {
+      // nothing to do
+      if (!scaleParameter.Fixed) {
+        scaleParameter.SetValue(new DoubleValue(Math.Exp(2 * hyp[0])));
+      } else {
+        if (hyp.Length > 0) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceNoise", "hyp");
+      }
+    }
+    public void SetData(double[,] x, double[,] xt) {
+      // nothing to do
+    public double GetCovariance(double[,] x, int i, int j, IEnumerable<int> columnIndices) {
+      return sf2;
+    }
+    public double GetCovariance(int i, int j) {
+      if (i == j) return sf2;
+      else return 0.0;
+    public IEnumerable<double> GetGradient(double[,] x, int i, int j, IEnumerable<int> columnIndices) {
+      yield return 2 * sf2;
+    }
+    public double GetGradient(int i, int j, int k) {
+      if (k != 0) throw new ArgumentException("CovarianceConst has only one hyperparameters", "k");
+      if (i == j)
+        return 2 * sf2;
+      else
+        return 0.0;
+    public double GetCrossCovariance(double[,] x, double[,] xt, int i, int j, IEnumerable<int> columnIndices) {
+      return 0.0;
+    }
+  }

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovariancePeriodic.cs

-                      r8477
+                      r8742
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Data;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 …
   [StorableClass]
   [Item(Name = "CovariancePeriodic", Description = "Periodic covariance function for Gaussian processes.")]
+  public class CovariancePeriodic : Item, ICovarianceFunction {
+  public sealed class CovariancePeriodic : ParameterizedNamedItem, ICovarianceFunction {
     [Storable]
     private double[,] x;
+    private double scale;
     [Storable]
+    private double[,] xt;
+    private readonly HyperParameter<DoubleValue> scaleParameter;
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return scaleParameter; }
+    }
     [Storable]
+    private double sf2;
+    public double Scale { get { return sf2; } }
+    private double inverseLength;
     [Storable]
+    private double l;
+    public double Length { get { return l; } }
+    private readonly HyperParameter<DoubleValue> inverseLengthParameter;
+    public IValueParameter<DoubleValue> InverseLengthParameter {
+      get { return inverseLengthParameter; }
+    }
     [Storable]
+    private double p;
+    public double Period { get { return p; } }
+    private double period;
+    [Storable]
+    private readonly HyperParameter<DoubleValue> periodParameter;
+    public IValueParameter<DoubleValue> PeriodParameter {
+      get { return periodParameter; }
+    }
-    private bool symmetric;
+    private double[,] sd;
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return 3;
+    [StorableConstructor]
+    private CovariancePeriodic(bool deserializing) : base(deserializing) { }
+    private CovariancePeriodic(CovariancePeriodic original, Cloner cloner)
+      : base(original, cloner) {
+      this.scaleParameter = cloner.Clone(original.scaleParameter);
+      this.inverseLengthParameter = cloner.Clone(original.inverseLengthParameter);
+      this.periodParameter = cloner.Clone(original.periodParameter);
+      this.scale = original.scale;
+      this.inverseLength = original.inverseLength;
+      this.period = original.period;
+      RegisterEvents();
+    }
+    [StorableConstructor]
+    protected CovariancePeriodic(bool deserializing) : base(deserializing) { }
+    protected CovariancePeriodic(CovariancePeriodic original, Cloner cloner)
+      : base(original, cloner) {
+      if (original.x != null) {
+        x = new double[original.x.GetLength(0), original.x.GetLength(1)];
+        Array.Copy(original.x, x, x.Length);
+        xt = new double[original.xt.GetLength(0), original.xt.GetLength(1)];
+        Array.Copy(original.xt, xt, xt.Length);
+      }
+      sf2 = original.sf2;
+      l = original.l;
+      p = original.p;
+      symmetric = original.symmetric;
+    }
     public CovariancePeriodic()
       : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+      scaleParameter = new HyperParameter<DoubleValue>("Scale", "The scale of the periodic covariance function.");
+      inverseLengthParameter = new HyperParameter<DoubleValue>("InverseLength", "The inverse length parameter for the periodic covariance function.");
+      periodParameter = new HyperParameter<DoubleValue>("Period", "The period parameter for the periodic covariance function.");
+      Parameters.Add(scaleParameter);
+      Parameters.Add(inverseLengthParameter);
+      Parameters.Add(periodParameter);
+      RegisterEvents();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      RegisterEvents();
+    }
 …
+    }
+    public void SetParameter(double[] hyp) {
+      if (hyp.Length != 3) throw new ArgumentException();
+      this.l = Math.Exp(hyp[0]);
+      this.p = Math.Exp(hyp[1]);
+      this.sf2 = Math.Exp(2 * hyp[2]);
+      // sf2 = Math.Min(10E6, sf2); // upper limit for the scale
+      sd = null;
+    }
+    public void SetData(double[,] x) {
+      SetData(x, x);
+      this.symmetric = true;
+    // caching
+    private void RegisterEvents() {
+      Util.AttachValueChangeHandler<DoubleValue, double>(scaleParameter, () => { scale = scaleParameter.Value.Value; });
+      Util.AttachValueChangeHandler<DoubleValue, double>(inverseLengthParameter, () => { inverseLength = inverseLengthParameter.Value.Value; });
+      Util.AttachValueChangeHandler<DoubleValue, double>(periodParameter, () => { period = periodParameter.Value.Value; });
+    }
+    public void SetData(double[,] x, double[,] xt) {
+      this.x = x;
+      this.xt = xt;
+      this.symmetric = false;
+      sd = null;
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return
+        (new[] { scaleParameter, inverseLengthParameter, periodParameter }).Count(p => !p.Fixed);
+    }
+    public double GetCovariance(int i, int j) {
+      if (sd == null) CalculateSquaredDistances();
+      double k = sd[i, j];
+      k = Math.PI * k / p;
+      k = Math.Sin(k) / l;
+    public void SetParameter(double[] hyp) {
+      int i = 0;
+      if (!inverseLengthParameter.Fixed) {
+        inverseLengthParameter.SetValue(new DoubleValue(1.0 / Math.Exp(hyp[i])));
+        i++;
+      }
+      if (!periodParameter.Fixed) {
+        periodParameter.SetValue(new DoubleValue(Math.Exp(hyp[i])));
+        i++;
+      }
+      if (!scaleParameter.Fixed) {
+        scaleParameter.SetValue(new DoubleValue(Math.Exp(2 * hyp[i])));
+        i++;
+      }
+      if (hyp.Length != i) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovariancePeriod", "hyp");
+    }
+    public double GetCovariance(double[,] x, int i, int j, IEnumerable<int> columnIndices) {
+      double k = i == j ? 0.0 : GetDistance(x, x, i, j, columnIndices);
+      k = Math.PI * k / period;
+      k = Math.Sin(k) * inverseLength;
       k = k * k;
       return sf2 * Math.Exp(-2.0 * k);
+      return scale * Math.Exp(-2.0 * k);
+    }
+    public double GetGradient(int i, int j, int k) {
+      double v = Math.PI * sd[i, j] / p;
+      switch (k) {
+        case 0: {
+            double newK = Math.Sin(v) / l;
+            newK = newK * newK;
+            return 4 * sf2 * Math.Exp(-2 * newK) * newK;
+          }
+        case 1: {
+            double r = Math.Sin(v) / l;
+            return 4 * sf2 / l * Math.Exp(-2 * r * r) * r * Math.Cos(v) * v;
+          }
+        case 2: {
+            double newK = Math.Sin(v) / l;
+            newK = newK * newK;
+            return 2 * sf2 * Math.Exp(-2 * newK);
+          }
+        default: {
+            throw new ArgumentException("CovariancePeriodic only has three hyperparameters.", "k");
+          }
+      }
+    public IEnumerable<double> GetGradient(double[,] x, int i, int j, IEnumerable<int> columnIndices) {
+      double v = i == j ? 0.0 : Math.PI * GetDistance(x, x, i, j, columnIndices) / period;
+      double gradient = Math.Sin(v) * inverseLength;
+      gradient *= gradient;
+      yield return 4.0 * scale * Math.Exp(-2.0 * gradient) * gradient;
+      double r = Math.Sin(v) * inverseLength;
+      yield return 4.0 * scale * inverseLength * Math.Exp(-2 * r * r) * r * Math.Cos(v) * v;
+      yield return 2.0 * scale * Math.Exp(-2 * gradient);
+    }
     private void CalculateSquaredDistances() {
       if (x.GetLength(1) != xt.GetLength(1)) throw new InvalidOperationException();
       int rows = x.GetLength(0);
       int cols = xt.GetLength(0);
       sd = new double[rows, cols];
+    public double GetCrossCovariance(double[,] x, double[,] xt, int i, int j, IEnumerable<int> columnIndices) {
+      double k = GetDistance(x, xt, i, j, columnIndices);
+      k = Math.PI * k / period;
+      k = Math.Sin(k) * inverseLength;
+      k = k * k;
+      if (symmetric) {
+        for (int i = 0; i < rows; i++) {
+          for (int j = i; j < cols; j++) {
+            sd[i, j] = Math.Sqrt(Util.SqrDist(Util.GetRow(x, i), Util.GetRow(x, j)));
+            sd[j, i] = sd[i, j];
+          }
+        }
+      } else {
+        for (int i = 0; i < rows; i++) {
+          for (int j = 0; j < cols; j++) {
+            sd[i, j] = Math.Sqrt(Util.SqrDist(Util.GetRow(x, i), Util.GetRow(xt, j)));
+          }
+        }
+      }
+      return scale * Math.Exp(-2.0 * k);
+    }
+    private double GetDistance(double[,] x, double[,] xt, int i, int j, IEnumerable<int> columnIndices) {
+      return Math.Sqrt(Util.SqrDist(x, i, xt, j, 1, columnIndices));
+    }
+  }

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceSum.cs

-                      r8477
+                      r8742
   [Item(Name = "CovarianceSum",
     Description = "Sum covariance function for Gaussian processes.")]
   public class CovarianceSum : Item, ICovarianceFunction {
+  public sealed class CovarianceSum : Item, ICovarianceFunction {
     [Storable]
     private ItemList<ICovarianceFunction> terms;
 …
     [StorableConstructor]
     protected CovarianceSum(bool deserializing)
+    private CovarianceSum(bool deserializing)
       : base(deserializing) {
+    }
     protected CovarianceSum(CovarianceSum original, Cloner cloner)
+    private CovarianceSum(CovarianceSum original, Cloner cloner)
       : base(original, cloner) {
       this.terms = cloner.Clone(original.terms);
       this.numberOfVariables = original.numberOfVariables;
-      AttachEventHandlers();
+    }
 …
       : base() {
       this.terms = new ItemList<ICovarianceFunction>();
-      AttachEventHandlers();
+    }
-    private void AttachEventHandlers() {
-      this.terms.CollectionReset += (sender, args) => ClearCache();
-      this.terms.ItemsAdded += (sender, args) => ClearCache();
-      this.terms.ItemsRemoved += (sender, args) => ClearCache();
-      this.terms.ItemsReplaced += (sender, args) => ClearCache();
-      this.terms.ItemsMoved += (sender, args) => ClearCache();
+    }
 …
     public void SetParameter(double[] hyp) {
+      if (terms.Count == 0) throw new ArgumentException("At least one term is needed for sum covariance function.");
       int offset = 0;
       foreach (var t in terms) {
 …
+      }
+    }
+    public void SetData(double[,] x) {
+      SetData(x, x);
+    public double GetCovariance(double[,] x, int i, int j, IEnumerable<int> columnIndices) {
+      return terms.Select(t => t.GetCovariance(x, i, j, columnIndices)).Sum();
+    }
+    public void SetData(double[,] x, double[,] xt) {
+      foreach (var t in terms) {
+        t.SetData(x, xt);
+      }
+    public IEnumerable<double> GetGradient(double[,] x, int i, int j, IEnumerable<int> columnIndices) {
+      return terms.Select(t => t.GetGradient(x, i, j, columnIndices)).Aggregate(Enumerable.Concat);
+    }
+    public double GetCovariance(int i, int j) {
+      return terms.Select(t => t.GetCovariance(i, j)).Sum();
+    }
+    private Dictionary<int, Tuple<int, int>> cachedParameterMap;
+    public double GetGradient(int i, int j, int k) {
+      if (cachedParameterMap == null) {
+        CalculateParameterMap();
+      }
+      int ti = cachedParameterMap[k].Item1;
+      k = cachedParameterMap[k].Item2;
+      return terms[ti].GetGradient(i, j, k);
+    }
+    private void ClearCache() {
+      cachedParameterMap = null;
+    }
+    private void CalculateParameterMap() {
+      cachedParameterMap = new Dictionary<int, Tuple<int, int>>();
+      int k = 0;
+      for (int ti = 0; ti < terms.Count; ti++) {
+        for (int ti_k = 0; ti_k < terms[ti].GetNumberOfParameters(numberOfVariables); ti_k++) {
+          cachedParameterMap[k++] = Tuple.Create(ti, ti_k);
+        }
+      }
+    public double GetCrossCovariance(double[,] x, double[,] xt, int i, int j, IEnumerable<int> columnIndices) {
+      return terms.Select(t => t.GetCrossCovariance(x, xt, i, j, columnIndices)).Sum();
+    }
+  }

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessHyperparameterInitializer.cs

r8477	r8742
91	91	var rand = RandomParameter.ActualValue;
92	92	for (int i = 0; i < r.Length; i++)
93		r[i] = rand.NextDouble() * ~~2 - 1~~;
	93	r[i] = rand.NextDouble() * 10 - 5;
94	94
95	95	HyperparameterParameter.ActualValue = r;

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs

-                      r8477
+                      r8742
     [Storable]
+    private double[] hyperparameterGradients;
+    public double[] HyperparameterGradients {
+      get {
+        var copy = new double[hyperparameterGradients.Length];
+        Array.Copy(hyperparameterGradients, copy, copy.Length);
+        return copy;
+      }
+    }
+    [Storable]
     private ICovarianceFunction covarianceFunction;
     public ICovarianceFunction CovarianceFunction {
 …
     [Storable]
     private double sqrSigmaNoise;
+    public double SigmaNoise {
+      get { return Math.Sqrt(sqrSigmaNoise); }
+    }
     [Storable]
 …
       l = new double[n, n];
-      meanFunction.SetData(x);
-      covarianceFunction.SetData(x);
       // calculate means and covariances
       double[] m = meanFunction.GetMean(x);
       for (int i = 0; i < n; i++) {
         for (int j = i; j < n; j++) {
           l[j, i] = covarianceFunction.GetCovariance(i, j) / sqrSigmaNoise;
+          l[j, i] = covarianceFunction.GetCovariance(x, i, j) / sqrSigmaNoise;
           if (j == i) l[j, i] += 1.0;
+        }
 …
         alpha[i] = alpha[i] / sqrSigmaNoise;
       negativeLogLikelihood = 0.5 * Util.ScalarProd(ym, alpha) + diagSum + (n / 2.0) * Math.Log(2.0 * Math.PI * sqrSigmaNoise);
+    }
+    public double[] GetHyperparameterGradients() {
       // derivatives
-      int n = x.GetLength(0);
       int nAllowedVariables = x.GetLength(1);
-      int info;
       alglib.matinvreport matInvRep;
       double[,] lCopy = new double[l.GetLength(0), l.GetLength(1)];
 …
       if (covGradients.Length > 0) {
         for (int i = 0; i < n; i++) {
+          for (int j = 0; j < i; j++) {
+            var g = covarianceFunction.GetGradient(x, i, j).ToArray();
+            for (int k = 0; k < covGradients.Length; k++) {
+              covGradients[k] += lCopy[i, j] * g[k];
+            }
+          }
+          var gDiag = covarianceFunction.GetGradient(x, i, i).ToArray();
           for (int k = 0; k < covGradients.Length; k++) {
+            for (int j = 0; j < i; j++) {
+              covGradients[k] += lCopy[i, j] * covarianceFunction.GetGradient(i, j, k);
+            }
+            covGradients[k] += 0.5 * lCopy[i, i] * covarianceFunction.GetGradient(i, i, k);
+            // diag
+            covGradients[k] += 0.5 * lCopy[i, i] * gDiag[k];
+          }
+        }
+      }
       return
+      hyperparameterGradients =
         meanGradients
         .Concat(covGradients)
         .Concat(new double[] { noiseGradient }).ToArray();
+    }
 …
+    }
     public GaussianProcessRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
       return new GaussianProcessRegressionSolution(this, problemData);
+      return new GaussianProcessRegressionSolution(this, new RegressionProblemData(problemData));
+    }
     IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
 …
+    }
     #endregion
     private IEnumerable<double> GetEstimatedValuesHelper(Dataset dataset, IEnumerable<int> rows) {
 …
       int newN = newX.GetLength(0);
       int n = x.GetLength(0);
-      // var predMean = new double[newN];
-      // predVar = new double[newN];
-      // var kss = new double[newN];
       var Ks = new double[newN, n];
-      //double[,] sWKs = new double[n, newN];
-      // double[,] v;
-      // for stddev
-      //covarianceFunction.SetParameter(covHyp, newX);
-      //kss = covarianceFunction.GetDiagonalCovariances();
-      covarianceFunction.SetData(x, newX);
-      meanFunction.SetData(newX);
       var ms = meanFunction.GetMean(newX);
       for (int i = 0; i < newN; i++) {
         for (int j = 0; j < n; j++) {
+          Ks[i, j] = covarianceFunction.GetCovariance(j, i);
+          //sWKs[j, i] = Ks[i, j] / Math.Sqrt(sqrSigmaNoise);
+        }
+      }
+      // for stddev
+      // alglib.rmatrixsolvem(l, n, sWKs, newN, true, out info, out denseSolveRep, out v);
+          Ks[i, j] = covarianceFunction.GetCrossCovariance(x, newX, j, i);
+        }
+      }
       return Enumerable.Range(0, newN)
         .Select(i => ms[i] + Util.ScalarProd(Util.GetRow(Ks, i), alpha));
-      //for (int i = 0; i < newN; i++) {
-      //  // predMean[i] = ms[i] + prod(GetRow(Ks, i), alpha);
-      //  // var sumV2 = prod(GetCol(v, i), GetCol(v, i));
-      //  // predVar[i] = kss[i] - sumV2;
-      //}
+    }
 …
       // for stddev
-      covarianceFunction.SetData(newX);
       for (int i = 0; i < newN; i++)
+        kss[i] = covarianceFunction.GetCovariance(i, i);
+      covarianceFunction.SetData(x, newX);
+        kss[i] = covarianceFunction.GetCovariance(newX, i, i);
       for (int i = 0; i < newN; i++) {
         for (int j = 0; j < n; j++) {
           sWKs[j, i] = covarianceFunction.GetCovariance(j, i) / Math.Sqrt(sqrSigmaNoise);
+          sWKs[j, i] = covarianceFunction.GetCrossCovariance(x, newX, j, i) / Math.Sqrt(sqrSigmaNoise);
+        }
+      }
       // for stddev
+      int info;
+      alglib.densesolverreport denseSolveRep;
+      double[,] v;
+      alglib.rmatrixsolvem(l, n, sWKs, newN, false, out info, out denseSolveRep, out v);
+      alglib.ablas.rmatrixlefttrsm(n, newN, l, 0, 0, false, false, 0, ref sWKs, 0, 0);
       for (int i = 0; i < newN; i++) {
         var sumV = Util.ScalarProd(Util.GetCol(v, i), Util.GetCol(v, i));
+        var sumV = Util.ScalarProd(Util.GetCol(sWKs, i), Util.GetCol(sWKs, i));
         kss[i] -= sumV;
         if (kss[i] < 0) kss[i] = 0;

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegression.cs

-                      r8477
+                      r8742
 using System;
-using System.Collections.Generic;
-using System.Linq;
 using HeuristicLab.Algorithms.GradientDescent;
 using HeuristicLab.Common;
 …
 using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using HeuristicLab.PluginInfrastructure;
 using HeuristicLab.Problems.DataAnalysis;
 …
     #region parameter properties
     public IConstrainedValueParameter<IMeanFunction> MeanFunctionParameter {
       get { return (IConstrainedValueParameter<IMeanFunction>)Parameters[MeanFunctionParameterName]; }
+    public IValueParameter<IMeanFunction> MeanFunctionParameter {
+      get { return (IValueParameter<IMeanFunction>)Parameters[MeanFunctionParameterName]; }
+    }
     public IConstrainedValueParameter<ICovarianceFunction> CovarianceFunctionParameter {
       get { return (IConstrainedValueParameter<ICovarianceFunction>)Parameters[CovarianceFunctionParameterName]; }
+    public IValueParameter<ICovarianceFunction> CovarianceFunctionParameter {
+      get { return (IValueParameter<ICovarianceFunction>)Parameters[CovarianceFunctionParameterName]; }
+    }
     public IValueParameter<IntValue> MinimizationIterationsParameter {
 …
       Problem = new RegressionProblem();
+      List<IMeanFunction> meanFunctions = ApplicationManager.Manager.GetInstances<IMeanFunction>().ToList();
+      List<ICovarianceFunction> covFunctions = ApplicationManager.Manager.GetInstances<ICovarianceFunction>().ToList();
+      Parameters.Add(new ConstrainedValueParameter<IMeanFunction>(MeanFunctionParameterName, "The mean function to use.",
+        new ItemSet<IMeanFunction>(meanFunctions), meanFunctions.OfType<MeanConst>().First()));
+      Parameters.Add(new ConstrainedValueParameter<ICovarianceFunction>(CovarianceFunctionParameterName, "The covariance function to use.",
+        new ItemSet<ICovarianceFunction>(covFunctions), covFunctions.OfType<CovarianceSEiso>().First()));
+      Parameters.Add(new ValueParameter<IMeanFunction>(MeanFunctionParameterName, "The mean function to use.", new MeanConst()));
+      Parameters.Add(new ValueParameter<ICovarianceFunction>(CovarianceFunctionParameterName, "The covariance function to use.", new CovarianceSquaredExponentialIso()));
       Parameters.Add(new ValueParameter<IntValue>(MinimizationIterationsParameterName, "The number of iterations for likelihood optimization with LM-BFGS.", new IntValue(20)));
       Parameters.Add(new ValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs

r8477	r8742
65	65	ModelParameter.ActualValue = model;
66	66	NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(model.NegativeLogLikelihood);
67		HyperparameterGradientsParameter.ActualValue = new RealVector(model.~~GetHyperparameterGradients()~~);
	67	HyperparameterGradientsParameter.ActualValue = new RealVector(model.HyperparameterGradients);
68	68	return base.Apply();
69	69	}

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolution.cs

-                      r8477
+                      r8742
   [StorableClass]
   public sealed class GaussianProcessRegressionSolution : RegressionSolution, IGaussianProcessSolution {
+    private new readonly Dictionary<int, double> evaluationCache;
     public new IGaussianProcessModel Model {
 …
     [StorableConstructor]
+    private GaussianProcessRegressionSolution(bool deserializing) : base(deserializing) { }
+    private GaussianProcessRegressionSolution(bool deserializing)
+      : base(deserializing) {
+      evaluationCache = new Dictionary<int, double>();
+    }
     private GaussianProcessRegressionSolution(GaussianProcessRegressionSolution original, Cloner cloner)
       : base(original, cloner) {
+      evaluationCache = new Dictionary<int, double>(original.evaluationCache);
+    }
     public GaussianProcessRegressionSolution(IGaussianProcessModel model, IRegressionProblemData problemData)
       : base(model, problemData) {
+      evaluationCache = new Dictionary<int, double>(problemData.Dataset.Rows);
       RecalculateResults();
+    }
 …
     public IEnumerable<double> GetEstimatedVariance(IEnumerable<int> rows) {
+      return Model.GetEstimatedVariance(ProblemData.Dataset, rows);
+      var rowsToEvaluate = rows.Except(evaluationCache.Keys);
+      var rowsEnumerator = rowsToEvaluate.GetEnumerator();
+      var valuesEnumerator = Model.GetEstimatedVariance(ProblemData.Dataset, rowsToEvaluate).GetEnumerator();
+      while (rowsEnumerator.MoveNext() & valuesEnumerator.MoveNext()) {
+        evaluationCache.Add(rowsEnumerator.Current, valuesEnumerator.Current);
+      }
+      return rows.Select(row => evaluationCache[row]);
+    }
+    protected override void OnModelChanged() {
+      evaluationCache.Clear();
+      base.OnModelChanged();
+    }
+    protected override void OnProblemDataChanged() {
+      evaluationCache.Clear();
+      base.OnProblemDataChanged();
+    }
+  }

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs

-                      r8477
+                      r8742
     public override IOperation Apply() {
+      var m = (IGaussianProcessModel)ModelParameter.ActualValue.Clone();
+      var data = (IRegressionProblemData)ProblemDataParameter.ActualValue.Clone();
+      var s = new GaussianProcessRegressionSolution(m, data);
+      if (ModelParameter.ActualValue != null) {
+        var m = (IGaussianProcessModel)ModelParameter.ActualValue.Clone();
+        var data = (IRegressionProblemData)ProblemDataParameter.ActualValue.Clone();
+        var s = new GaussianProcessRegressionSolution(m, data);
+      SolutionParameter.ActualValue = s;
+      var results = ResultsParameter.ActualValue;
+      if (!results.ContainsKey(SolutionParameterName)) {
+        results.Add(new Result(SolutionParameterName, "The Gaussian process regression solution", s));
+        results.Add(new Result(TrainingRSquaredResultName, "The Pearson's R² of the Gaussian process solution on the training partition.", new DoubleValue(s.TrainingRSquared)));
+        results.Add(new Result(TestRSquaredResultName, "The Pearson's R² of the Gaussian process solution on the test partition.", new DoubleValue(s.TestRSquared)));
+      } else {
+        results[SolutionParameterName].Value = s;
+        results[TrainingRSquaredResultName].Value = new DoubleValue(s.TrainingRSquared);
+        results[TestRSquaredResultName].Value = new DoubleValue(s.TestRSquared);
+        SolutionParameter.ActualValue = s;
+        var results = ResultsParameter.ActualValue;
+        if (!results.ContainsKey(SolutionParameterName)) {
+          results.Add(new Result(SolutionParameterName, "The Gaussian process regression solution", s));
+          results.Add(new Result(TrainingRSquaredResultName,
+                                 "The Pearson's R² of the Gaussian process solution on the training partition.",
+                                 new DoubleValue(s.TrainingRSquared)));
+          results.Add(new Result(TestRSquaredResultName,
+                                 "The Pearson's R² of the Gaussian process solution on the test partition.",
+                                 new DoubleValue(s.TestRSquared)));
+        } else {
+          results[SolutionParameterName].Value = s;
+          results[TrainingRSquaredResultName].Value = new DoubleValue(s.TrainingRSquared);
+          results[TestRSquaredResultName].Value = new DoubleValue(s.TestRSquared);
+        }
+      }
       return base.Apply();

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/ICovarianceFunction.cs

-                      r8477
+                      r8742
 #endregion
+using System.Collections.Generic;
 using HeuristicLab.Core;
 …
     int GetNumberOfParameters(int numberOfVariables);
     void SetParameter(double[] hyp);
+    void SetData(double[,] x);
+    void SetData(double[,] x, double[,] xt);
+    double GetCovariance(int i, int j);
+    double GetGradient(int i, int j, int k);
+    double GetCovariance(double[,] x, int i, int j, IEnumerable<int> columnIndices = null);
+    IEnumerable<double> GetGradient(double[,] x, int i, int j, IEnumerable<int> columnIndices = null);
+    double GetCrossCovariance(double[,] x, double[,] xt, int i, int j, IEnumerable<int> columnIndices = null);
+  }
+}

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/IMeanFunction.cs

r8416	r8742
25	25	int GetNumberOfParameters(int numberOfVariables);
26	26	void SetParameter(double[] hyp);
27		~~void SetData(double[,] x);~~
28	27	double[] GetMean(double[,] x);
29	28	double[] GetGradients(int k, double[,] x);

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanConst.cs

-                      r8477
+                      r8742
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Data;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 …
   [StorableClass]
   [Item(Name = "MeanConst", Description = "Constant mean function for Gaussian processes.")]
   public class MeanConst : Item, IMeanFunction {
+  public sealed class MeanConst : ParameterizedNamedItem, IMeanFunction {
     [Storable]
     private double c;
+    public double Value { get { return c; } }
+    [Storable]
+    private readonly HyperParameter<DoubleValue> valueParameter;
+    public IValueParameter<DoubleValue> ValueParameter { get { return valueParameter; } }
-    public int GetNumberOfParameters(int numberOfVariables) {
-      return 1;
+    }
     [StorableConstructor]
     protected MeanConst(bool deserializing) : base(deserializing) { }
     protected MeanConst(MeanConst original, Cloner cloner)
+    private MeanConst(bool deserializing) : base(deserializing) { }
+    private MeanConst(MeanConst original, Cloner cloner)
       : base(original, cloner) {
       this.c = original.c;
+      this.valueParameter = cloner.Clone(original.valueParameter);
+      RegisterEvents();
+    }
     public MeanConst()
       : base() {
+      this.name = ItemName;
+      this.description = ItemDescription;
+      this.valueParameter = new HyperParameter<DoubleValue>("Value", "The constant value for the constant mean function.");
+      Parameters.Add(valueParameter);
+      RegisterEvents();
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanConst(this, cloner);
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      RegisterEvents();
+    }
+    private void RegisterEvents() {
+      Util.AttachValueChangeHandler<DoubleValue, double>(valueParameter, () => { c = valueParameter.Value.Value; });
+    }
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return valueParameter.Fixed ? 0 : 1;
+    }
     public void SetParameter(double[] hyp) {
+      if (hyp.Length != 1) throw new ArgumentException("Only one hyper-parameter allowed for constant mean function.", "hyp");
+      this.c = hyp[0];
+    }
+    public void SetData(double[,] x) {
+      // nothing to do
+      if (!valueParameter.Fixed) {
+        valueParameter.SetValue(new DoubleValue(hyp[0]));
+      } else if (hyp.Length > 0)
+        throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for the constant mean function.", "hyp");
+    }
 …
       return Enumerable.Repeat(1.0, x.GetLength(0)).ToArray();
+    }
-    public override IDeepCloneable Clone(Cloner cloner) {
-      return new MeanConst(this, cloner);
+    }
+  }
+}

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanLinear.cs

-                      r8477
+                      r8742
  */
 #endregion
 using System;
 using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Data;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 …
   [StorableClass]
   [Item(Name = "MeanLinear", Description = "Linear mean function for Gaussian processes.")]
   public class MeanLinear : Item, IMeanFunction {
+  public sealed class MeanLinear : ParameterizedNamedItem, IMeanFunction {
     [Storable]
+    private double[] alpha;
+    public double[] Weights {
+      get {
+        if (alpha == null) return new double[0];
+        var copy = new double[alpha.Length];
+        Array.Copy(alpha, copy, copy.Length);
+        return copy;
+    private double[] weights;
+    [Storable]
+    private readonly HyperParameter<DoubleArray> weightsParameter;
+    public IValueParameter<DoubleArray> WeightsParameter { get { return weightsParameter; } }
+    [StorableConstructor]
+    private MeanLinear(bool deserializing) : base(deserializing) { }
+    private MeanLinear(MeanLinear original, Cloner cloner)
+      : base(original, cloner) {
+      if (original.weights != null) {
+        this.weights = new double[original.weights.Length];
+        Array.Copy(original.weights, weights, original.weights.Length);
+      }
+    }
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return numberOfVariables;
+    }
+    [StorableConstructor]
+    protected MeanLinear(bool deserializing) : base(deserializing) { }
+    protected MeanLinear(MeanLinear original, Cloner cloner)
+      : base(original, cloner) {
+      if (original.alpha != null) {
+        this.alpha = new double[original.alpha.Length];
+        Array.Copy(original.alpha, alpha, original.alpha.Length);
+      }
+      weightsParameter = cloner.Clone(original.weightsParameter);
+      RegisterEvents();
+    }
     public MeanLinear()
       : base() {
+      this.weightsParameter = new HyperParameter<DoubleArray>("Weights", "The weights parameter for the linear mean function.");
+      Parameters.Add(weightsParameter);
+      RegisterEvents();
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanLinear(this, cloner);
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      RegisterEvents();
+    }
+    private void RegisterEvents() {
+      Util.AttachArrayChangeHandler<DoubleArray, double>(weightsParameter, () => {
+        weights = weightsParameter.Value.ToArray();
+      });
+    }
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return weightsParameter.Fixed ? 0 : numberOfVariables;
+    }
     public void SetParameter(double[] hyp) {
+      this.alpha = new double[hyp.Length];
+      Array.Copy(hyp, alpha, hyp.Length);
+    }
+    public void SetData(double[,] x) {
+      // nothing to do
+      if (!weightsParameter.Fixed) {
+        weightsParameter.SetValue(new DoubleArray(hyp));
+      } else if (hyp.Length != 0) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for the linear mean function.", "hyp");
+    }
     public double[] GetMean(double[,] x) {
       // sanity check
       if (alpha.Length != x.GetLength(1)) throw new ArgumentException("The number of hyperparameters must match the number of variables for the linear mean function.");
+      if (weights.Length != x.GetLength(1)) throw new ArgumentException("The number of hyperparameters must match the number of variables for the linear mean function.");
       int cols = x.GetLength(1);
       int n = x.GetLength(0);
       return (from i in Enumerable.Range(0, n)
+              let rowVector = from j in Enumerable.Range(0, cols)
+                              select x[i, j]
+              select Util.ScalarProd(alpha, rowVector))
+              let rowVector = Enumerable.Range(0, cols).Select(j => x[i, j])
+              select Util.ScalarProd(weights, rowVector))
         .ToArray();
+    }
 …
       int n = x.GetLength(0);
       if (k > cols) throw new ArgumentException();
+      return (from r in Enumerable.Range(0, n)
+              select x[r, k]).ToArray();
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanLinear(this, cloner);
+      return (Enumerable.Range(0, n).Select(r => x[r, k])).ToArray();
+    }
+  }

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanSum.cs

-                      r8477
+                      r8742
   [StorableClass]
   [Item(Name = "MeanSum", Description = "Sum of mean functions for Gaussian processes.")]
   public class MeanSum : Item, IMeanFunction {
+  public sealed class MeanSum : Item, IMeanFunction {
     [Storable]
     private ItemList<IMeanFunction> terms;
 …
+    }
-    public int GetNumberOfParameters(int numberOfVariables) {
-      this.numberOfVariables = numberOfVariables;
-      return terms.Select(t => t.GetNumberOfParameters(numberOfVariables)).Sum();
+    }
     [StorableConstructor]
     protected MeanSum(bool deserializing) : base(deserializing) { }
     protected MeanSum(MeanSum original, Cloner cloner)
+    private MeanSum(bool deserializing) : base(deserializing) { }
+    private MeanSum(MeanSum original, Cloner cloner)
       : base(original, cloner) {
       this.terms = cloner.Clone(original.terms);
 …
     public MeanSum() {
       this.terms = new ItemList<IMeanFunction>();
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanSum(this, cloner);
+    }
+    public int GetNumberOfParameters(int numberOfVariables) {
+      this.numberOfVariables = numberOfVariables;
+      return terms.Select(t => t.GetNumberOfParameters(numberOfVariables)).Sum();
+    }
 …
         offset += numberOfParameters;
+      }
+    }
-    public void SetData(double[,] x) {
-      foreach (var t in terms) t.SetData(x);
+    }
 …
       return terms[i].GetGradients(k, x);
+    }
-    public override IDeepCloneable Clone(Cloner cloner) {
-      return new MeanSum(this, cloner);
+    }
+  }
+}

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanZero.cs

-                      r8416
+                      r8742
   [StorableClass]
   [Item(Name = "MeanZero", Description = "Constant zero mean function for Gaussian processes.")]
+  public class MeanZero : Item, IMeanFunction {
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return 0;
+    }
+  public sealed class MeanZero : Item, IMeanFunction {
     [StorableConstructor]
     protected MeanZero(bool deserializing) : base(deserializing) { }
     protected MeanZero(MeanZero original, Cloner cloner)
+    private MeanZero(bool deserializing) : base(deserializing) { }
+    private MeanZero(MeanZero original, Cloner cloner)
       : base(original, cloner) {
+    }
 …
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanZero(this, cloner);
+    }
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return 0;
+    }
     public void SetParameter(double[] hyp) {
       if (hyp.Length > 0) throw new ArgumentException("No hyper-parameters allowed for zero mean function.", "hyp");
+    }
-    public void SetData(double[,] x) {
-      // do nothing
+    }
 …
       return Enumerable.Repeat(0.0, x.GetLength(0)).ToArray();
+    }
-    public override IDeepCloneable Clone(Cloner cloner) {
-      return new MeanZero(this, cloner);
+    }
+  }
+}

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/Util.cs

-                      r8477
+                      r8742
 #endregion
+using System;
 using System.Collections.Generic;
 using System.Linq;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
 namespace HeuristicLab.Algorithms.DataAnalysis {
   public static class Util {
+  internal static class Util {
     public static double ScalarProd(IEnumerable<double> v, IEnumerable<double> u) {
       return v.Zip(u, (vi, ui) => vi * ui).Sum();
+    }
+    public static double SqrDist(IEnumerable<double> x, IEnumerable<double> y) {
+      return x.Zip(y, (a, b) => (a - b) * (a - b)).Sum();
+    }
 …
+    }
+    public static double SqrDist(IEnumerable<double> x, IEnumerable<double> y) {
+      return x.Zip(y, (a, b) => (a - b) * (a - b)).Sum();
+    public static double SqrDist(double[,] x, int i, int j, double scale = 1.0, IEnumerable<int> columnIndices = null) {
+      return SqrDist(x, i, x, j, scale, columnIndices);
+    }
+    public static double SqrDist(double[,] x, int i, double[,] xt, int j, double scale = 1.0, IEnumerable<int> columnIndices = null) {
+      double ss = 0.0;
+      if (columnIndices == null) columnIndices = Enumerable.Range(0, x.GetLength(1));
+      foreach (int k in columnIndices) {
+        double d = x[i, k] - xt[j, k];
+        ss += d * d;
+      }
+      return scale * scale * ss;
+    }
+    public static double SqrDist(double[,] x, int i, int j, double[] scale, IEnumerable<int> columnIndices = null) {
+      return SqrDist(x, i, x, j, scale);
+    }
+    public static double SqrDist(double[,] x, int i, double[,] xt, int j, double[] scale, IEnumerable<int> columnIndices = null) {
+      double ss = 0.0;
+      if (columnIndices == null) columnIndices = Enumerable.Range(0, x.GetLength(1));
+      foreach (int k in columnIndices) {
+        double d = x[i, k] - xt[j, k];
+        ss += d * d * scale[k] * scale[k];
+      }
+      return ss;
+    }
+    public static double ScalarProd(double[,] x, int i, int j, double scale = 1.0, IEnumerable<int> columnIndices = null) {
+      return ScalarProd(x, i, x, j, scale, columnIndices);
+    }
+    public static double ScalarProd(double[,] x, int i, double[,] xt, int j, double scale = 1.0, IEnumerable<int> columnIndices = null) {
+      double sum = 0.0;
+      if (columnIndices == null) columnIndices = Enumerable.Range(0, x.GetLength(1));
+      foreach (int k in columnIndices) {
+        sum += x[i, k] * xt[j, k];
+      }
+      return scale * scale * sum;
+    }
+    public static double ScalarProd(double[,] x, int i, int j, double[] scale, IEnumerable<int> columnIndices = null) {
+      return ScalarProd(x, i, x, j, scale, columnIndices);
+    }
+    public static double ScalarProd(double[,] x, int i, double[,] xt, int j, double[] scale, IEnumerable<int> columnIndices = null) {
+      double sum = 0.0;
+      if (columnIndices == null) columnIndices = Enumerable.Range(0, x.GetLength(1));
+      foreach (int k in columnIndices) {
+        sum += x[i, k] * scale[k] * xt[j, k] * scale[k];
+      }
+      return sum;
+    }
 …
       return Enumerable.Range(0, rows).Select(r => x[r, c]);
+    }
+    public static void AttachValueChangeHandler<T, U>(IValueParameter<T> parameter, Action action)
+      where T : ValueTypeValue<U>
+      where U : struct {
+      parameter.ValueChanged += (sender, args) => {
+        if (parameter.Value != null) {
+          parameter.Value.ValueChanged += (s, a) => action();
+          action();
+        }
+      };
+      if (parameter.Value != null) {
+        parameter.Value.ValueChanged += (s, a) => action();
+      }
+    }
+    public static void AttachArrayChangeHandler<T, U>(IValueParameter<T> parameter, Action action)
+      where T : ValueTypeArray<U>
+      where U : struct {
+      parameter.ValueChanged += (sender, args) => {
+        if (parameter.Value != null) {
+          parameter.Value.ItemChanged += (s, a) => action();
+          parameter.Value.Reset += (s, a) => action();
+          action();
+        }
+      };
+      if (parameter.Value != null) {
+        parameter.Value.ItemChanged += (s, a) => action();
+        parameter.Value.Reset += (s, a) => action();
+      }
+    }
+  }
+}

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj

-                      r8477
+                      r8742
       <Private>False</Private>
     </Reference>
+    <Reference Include="HeuristicLab.Algorithms.Benchmarks-3.3">
+      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Algorithms.Benchmarks-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Algorithms.GradientDescent-3.3">
+      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Algorithms.GradientDescent-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Analysis-3.3">
+      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Analysis-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Collections-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Collections-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Common-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Common-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Common.Resources-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Common.Resources-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Core-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Core-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Data-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Data-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Encodings.RealVectorEncoding-3.3">
+      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Encodings.RealVectorEncoding-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Operators-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Operators-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Optimization-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Optimization-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Parameters-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Parameters-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Persistence-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Persistence-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.PluginInfrastructure-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.PluginInfrastructure-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Problems.DataAnalysis.Symbolic.Classification-3.4">
+      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Problems.DataAnalysis.Symbolic.Classification-3.4.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4">
+      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Problems.Instances-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Problems.Instances-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Random-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Random-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="LibSVM-1.6.3, Version=1.6.3.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\..\..\trunk\sources\bin\LibSVM-1.6.3.dll</HintPath>
+    <Reference Include="HeuristicLab.Algorithms.GradientDescent-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
+    <Reference Include="HeuristicLab.Analysis-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
+    <Reference Include="HeuristicLab.Collections-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
+    <Reference Include="HeuristicLab.Common-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
+    <Reference Include="HeuristicLab.Common.Resources-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
+    <Reference Include="HeuristicLab.Core-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
+    <Reference Include="HeuristicLab.Data-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
+    <Reference Include="HeuristicLab.Encodings.RealVectorEncoding-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
+    <Reference Include="HeuristicLab.Operators-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
+    <Reference Include="HeuristicLab.Optimization-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
+    <Reference Include="HeuristicLab.Parameters-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
+    <Reference Include="HeuristicLab.Persistence-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
+    <Reference Include="HeuristicLab.PluginInfrastructure-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
+    <Reference Include="HeuristicLab.Problems.DataAnalysis.Symbolic.Classification-3.4, Version=3.4.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
+    <Reference Include="HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4, Version=3.4.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
+    <Reference Include="HeuristicLab.Problems.Instances-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
+    <Reference Include="HeuristicLab.Random-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL" />
+    <Reference Include="LibSVM-3.12, Version=3.12.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\..\..\trunk\sources\bin\LibSVM-3.12.dll</HintPath>
       <Private>False</Private>
     </Reference>
 …
     </Compile>
     <Compile Include="FixedDataAnalysisAlgorithm.cs" />
+    <Compile Include="GaussianProcess\CovarianceRQiso.cs" />
+    <Compile Include="GaussianProcess\CovarianceMask.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessClassificationSolutionCreator.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessClassificationModelCreator.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessClassification.cs" />
+    <Compile Include="GaussianProcess\CovarianceProduct.cs" />
+    <Compile Include="GaussianProcess\CovarianceScale.cs" />
+    <Compile Include="GaussianProcess\CovarianceRationalQuadraticArd.cs" />
+    <Compile Include="GaussianProcess\CovarianceRationalQuadraticIso.cs" />
+    <Compile Include="GaussianProcess\CovarianceSquaredExponentialArd.cs" />
+    <Compile Include="GaussianProcess\CovarianceSquaredExponentialIso.cs" />
+    <Compile Include="GaussianProcess\HyperParameter.cs" />
+    <Compile Include="GaussianProcess\CovarianceMaternIso.cs" />
+    <Compile Include="GaussianProcess\CovarianceLinearArd.cs" />
     <Compile Include="GaussianProcess\CovarianceNoise.cs" />
     <Compile Include="GaussianProcess\CovarianceConst.cs" />
     <Compile Include="GaussianProcess\MeanProd.cs" />
+    <Compile Include="GaussianProcess\MeanProduct.cs" />
     <Compile Include="GaussianProcess\MeanSum.cs" />
-    <Compile Include="GaussianProcess\CovarianceProd.cs" />
     <Compile Include="GaussianProcess\CovarianceSum.cs" />
     <Compile Include="GaussianProcess\CovariancePeriodic.cs" />
 …
     <Compile Include="GaussianProcess\MeanConst.cs" />
     <Compile Include="GaussianProcess\IMeanFunction.cs" />
-    <Compile Include="GaussianProcess\CovarianceSEard.cs" />
-    <Compile Include="GaussianProcess\CovarianceSEiso.cs" />
     <Compile Include="GaussianProcess\GaussianProcessModel.cs" />
     <Compile Include="GaussianProcess\GaussianProcessRegression.cs" />
 …
       <Name>HeuristicLab.Problems.DataAnalysis-3.4</Name>
     </ProjectReference>
-    <ProjectReference Include="..\..\HeuristicLab.Random\3.3\HeuristicLab.Random-3.3.csproj">
-      <Project>{F4539FB6-4708-40C9-BE64-0A1390AEA197}</Project>
-      <Name>HeuristicLab.Random-3.3</Name>
-      <Private>False</Private>
-    </ProjectReference>
   </ItemGroup>
   <ItemGroup>
 …
   -->
   <PropertyGroup>
     <PreBuildEvent>set Path=%25Path%25;$(ProjectDir);$(SolutionDir)
+    <PreBuildEvent Condition=" '$(OS)' == 'Windows_NT' ">set Path=%25Path%25;$(ProjectDir);$(SolutionDir)
 set ProjectDir=$(ProjectDir)
 set SolutionDir=$(SolutionDir)
 …
 call PreBuildEvent.cmd
 </PreBuildEvent>
+    <PreBuildEvent Condition=" '$(OS)' != 'Windows_NT' ">
+export ProjectDir=$(ProjectDir)
+export SolutionDir=$(SolutionDir)
+$SolutionDir/PreBuildEvent.sh
+</PreBuildEvent>
   </PropertyGroup>
 </Project>

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModel.cs

-                      r8477
+                      r8742
   public interface IGaussianProcessModel : IRegressionModel {
     double NegativeLogLikelihood { get; }
+    double SigmaNoise { get; }
     IMeanFunction MeanFunction { get; }
     ICovarianceFunction CovarianceFunction { get; }
     double[] GetHyperparameterGradients();
+    double[] HyperparameterGradients { get; }
     IEnumerable<double> GetEstimatedVariance(Dataset ds, IEnumerable<int> rows);

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineModel.cs

-                      r7259
+                      r8742
 #endregion
-using HeuristicLab.Optimization;
 using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Core;
+using System.Collections.Generic;
+using LibSVM;
 namespace HeuristicLab.Algorithms.DataAnalysis {
 …
   /// </summary>
   public interface ISupportVectorMachineModel : IDataAnalysisModel, IRegressionModel, IClassificationModel {
     SVM.Model Model { get; }
     SVM.RangeTransform RangeTransform { get; }
+    svm_model Model { get; }
+    RangeTransform RangeTransform { get; }
     Dataset SupportVectors { get; }
+  }

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearDiscriminantAnalysis.cs

r8430	r8742
111	111	IClassificationProblemData problemData,
112	112	IEnumerable<int> rows) {
113		return new SymbolicDiscriminantFunctionClassificationModel(tree, interpreter);
	113	var model = new SymbolicDiscriminantFunctionClassificationModel(tree, interpreter, new AccuracyMaximizationThresholdCalculator());
	114	model.RecalculateModelParameters(problemData, rows);
	115	return model;
114	116	}
115	117	}

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassificationSolution.cs

r7259	r8742
45	45	public MultinomialLogitClassificationSolution(IClassificationProblemData problemData, MultinomialLogitModel logitModel)
46	46	: base(logitModel, problemData) {
47		~~RecalculateResults();~~
48	47	}
49	48
…	…
51	50	return new MultinomialLogitClassificationSolution(this, cloner);
52	51	}
53
54		~~protected override void RecalculateResults() {~~
55		~~CalculateResults();~~
56		}
57	52	}
58	53	}

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitModel.cs

r7259	r8742
109	109
110	110	public MultinomialLogitClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
111		return new MultinomialLogitClassificationSolution(~~problemData~~, this);
	111	return new MultinomialLogitClassificationSolution(new ClassificationProblemData(problemData), this);
112	112	}
113	113	IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Matrix.cs

-                      r8471
+                      r8742
     public Matrix Transpose() {
+      var result = new Matrix(Transpose(values, Columns, Rows), Columns, Rows);
+      return result;
+      return new Matrix(Transpose(values, Columns, Rows), Columns, Rows);
+    }
 …
+    }
+    public double VectorLength() {
+      return Math.Sqrt(SquaredVectorLength());
+    }
+    public double SquaredVectorLength() {
+      if (Rows != 1) throw new ArgumentException("Length only works on vectors.");
+    public double EuclideanNorm() {
+      return Math.Sqrt(SumOfSquares());
+    }
+    public double SumOfSquares() {
       return values.Sum(x => x * x);
+    }
 …
       if (Rows != 1 || other.Rows != 1) throw new ArgumentException("OuterProduct can only be applied to vectors.");
       return Transpose().Multiply(other);
+    }
+    public IEnumerable<double> ColumnSums() {
+      return Transpose().RowSums();
+    }
+    public IEnumerable<double> RowSums() {
+      var sum = 0.0;
+      int counter = 0;
+      foreach (var v in values) {
+        sum += v;
+        counter++;
+        if (counter == Rows) {
+          yield return sum;
+          sum = 0.0;
+          counter = 0;
+        }
+      }
+    }

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaAlgorithm.cs

-                      r8471
+                      r8742
 namespace HeuristicLab.Algorithms.DataAnalysis {
   internal delegate void Reporter(double quality, double[] coefficients);
+  internal delegate void Reporter(double quality, double[] coefficients, double[] gradients);
   /// <summary>
   /// Neighborhood Components Analysis
   /// </summary>
+  [Item("Neighborhood Components Analysis (NCA)", "Implementation of Neighborhood Components Analysis based on the description of J. Goldberger, S. Roweis, G. Hinton, R. Salakhutdinov. 2005. Neighbourhood Component Analysis. Advances in Neural Information Processing Systems, 17. pp. 513-520.")]
+  [Item("Neighborhood Components Analysis (NCA)", @"Implementation of Neighborhood Components Analysis
+based on the description of J. Goldberger, S. Roweis, G. Hinton, R. Salakhutdinov. 2005.
+Neighbourhood Component Analysis. Advances in Neural Information Processing Systems, 17. pp. 513-520
+with additional regularizations described in Z. Yang, J. Laaksonen. 2007.
+Regularized Neighborhood Component Analysis. Lecture Notes in Computer Science, 4522. pp. 253-262.")]
   [Creatable("Data Analysis")]
   [StorableClass]
 …
       get { return (IFixedValueParameter<IntValue>)Parameters["Iterations"]; }
+    }
+    public IFixedValueParameter<DoubleValue> RegularizationParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters["Regularization"]; }
+    }
     #endregion
     #region Properties
     private int K {
+    public int K {
       get { return KParameter.Value.Value; }
       set { KParameter.Value.Value = value; }
+    }
     private int Dimensions {
+    public int Dimensions {
       get { return DimensionsParameter.Value.Value; }
       set { DimensionsParameter.Value.Value = value; }
+    }
     private int NeighborSamples {
+    public int NeighborSamples {
       get { return NeighborSamplesParameter.Value.Value; }
       set { NeighborSamplesParameter.Value.Value = value; }
+    }
     private int Iterations {
+    public int Iterations {
       get { return IterationsParameter.Value.Value; }
       set { IterationsParameter.Value.Value = value; }
+    }
+    public double Regularization {
+      get { return RegularizationParameter.Value.Value; }
+      set { RegularizationParameter.Value.Value = value; }
+    }
     #endregion
 …
     public NcaAlgorithm()
       : base() {
       Parameters.Add(new FixedValueParameter<IntValue>("K", "The K for the nearest neighbor.", new IntValue(1)));
+      Parameters.Add(new FixedValueParameter<IntValue>("K", "The K for the nearest neighbor.", new IntValue(3)));
       Parameters.Add(new FixedValueParameter<IntValue>("Dimensions", "The number of dimensions that NCA should reduce the data to.", new IntValue(2)));
       Parameters.Add(new ConstrainedValueParameter<INCAInitializer>("Initialization", "Which method should be used to initialize the matrix. Typically LDA (linear discriminant analysis) should provide a good estimate."));
+      Parameters.Add(new FixedValueParameter<IntValue>("NeighborSamples", "How many of the neighbors should be sampled in order to speed up the calculation. This should be at least the value of k and at most the number of training instances minus one.", new IntValue(50)));
+      Parameters.Add(new FixedValueParameter<IntValue>("Iterations", "How many iterations the conjugate gradient (CG) method should be allowed to perform. The method might still terminate earlier if a local optima has already been reached.", new IntValue(20)));
+      Parameters.Add(new FixedValueParameter<IntValue>("NeighborSamples", "How many of the neighbors should be sampled in order to speed up the calculation. This should be at least the value of k and at most the number of training instances minus one.", new IntValue(60)));
+      Parameters.Add(new FixedValueParameter<IntValue>("Iterations", "How many iterations the conjugate gradient (CG) method should be allowed to perform. The method might still terminate earlier if a local optima has already been reached.", new IntValue(50)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>("Regularization", "A non-negative paramter which can be set to increase generalization and avoid overfitting. If set to 0 the algorithm is similar to NCA as proposed by Goldberger et al.", new DoubleValue(0)));
       INCAInitializer defaultInitializer = null;
 …
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      if (!Parameters.ContainsKey("Regularization")) {
+        Parameters.Add(new FixedValueParameter<DoubleValue>("Regularization", "A non-negative paramter which can be set to increase generalization and avoid overfitting. If set to 0 the algorithm is similar to NCA as proposed by Goldberger et al.", new DoubleValue(0)));
+      }
+    }
     public override void Prepare() {
       if (Problem != null) base.Prepare();
 …
       var clonedProblem = (IClassificationProblemData)Problem.ProblemData.Clone();
+      var model = Train(clonedProblem, K, Dimensions, NeighborSamples, Iterations, initializer.Initialize(clonedProblem, Dimensions), ReportQuality, CancellationToken.None);
+      Results.Add(new Result("ClassificationSolution", "The classification solution.", model.CreateClassificationSolution(clonedProblem)));
+    }
+    public static INcaClassificationSolution CreateClassificationSolution(IClassificationProblemData data, int k, int dimensions, int neighborSamples, int iterations, INCAInitializer initializer) {
+      var model = Train(clonedProblem, K, Dimensions, NeighborSamples, Regularization, Iterations, initializer.Initialize(clonedProblem, Dimensions), ReportQuality, CancellationToken.None);
+      var solution = model.CreateClassificationSolution(clonedProblem);
+      if (!Results.ContainsKey("ClassificationSolution"))
+        Results.Add(new Result("ClassificationSolution", "The classification solution.", solution));
+      else Results["ClassificationSolution"].Value = solution;
+    }
+    public static INcaClassificationSolution CreateClassificationSolution(IClassificationProblemData data, int k, int dimensions, int neighborSamples, double regularization, int iterations, INCAInitializer initializer) {
       var clonedProblem = (IClassificationProblemData)data.Clone();
       var model = Train(clonedProblem, k, dimensions, neighborSamples, iterations, initializer);
+      var model = Train(clonedProblem, k, dimensions, neighborSamples, regularization, iterations, initializer);
       return model.CreateClassificationSolution(clonedProblem);
+    }
     public static INcaModel Train(IClassificationProblemData problemData, int k, int dimensions, int neighborSamples, int iterations, INCAInitializer initializer) {
       return Train(problemData, k, dimensions, neighborSamples, iterations, initializer.Initialize(problemData, dimensions), null, CancellationToken.None);
+    }
     public static INcaModel Train(IClassificationProblemData problemData, int k, int neighborSamples, int iterations, double[,] initalMatrix) {
+    public static INcaModel Train(IClassificationProblemData problemData, int k, int dimensions, int neighborSamples, double regularization, int iterations, INCAInitializer initializer) {
+      return Train(problemData, k, dimensions, neighborSamples, regularization, iterations, initializer.Initialize(problemData, dimensions), null, CancellationToken.None);
+    }
+    public static INcaModel Train(IClassificationProblemData problemData, int k, int neighborSamples, double regularization, int iterations, double[,] initalMatrix) {
       var matrix = new double[initalMatrix.Length];
       for (int i = 0; i < initalMatrix.GetLength(0); i++)
         for (int j = 0; j < initalMatrix.GetLength(1); j++)
           matrix[i * initalMatrix.GetLength(1) + j] = initalMatrix[i, j];
       return Train(problemData, k, initalMatrix.GetLength(1), neighborSamples, iterations, matrix, null, CancellationToken.None);
+    }
     private static INcaModel Train(IClassificationProblemData data, int k, int dimensions, int neighborSamples, int iterations, double[] matrix, Reporter reporter, CancellationToken cancellation) {
+      return Train(problemData, k, initalMatrix.GetLength(1), neighborSamples, regularization, iterations, matrix, null, CancellationToken.None);
+    }
+    private static INcaModel Train(IClassificationProblemData data, int k, int dimensions, int neighborSamples, double regularization, int iterations, double[] matrix, Reporter reporter, CancellationToken cancellation) {
       var scaling = new Scaling(data.Dataset, data.AllowedInputVariables, data.TrainingIndices);
       var scaledData = AlglibUtil.PrepareAndScaleInputMatrix(data.Dataset, data.AllowedInputVariables, data.TrainingIndices, scaling);
 …
       alglib.mincgsetcond(state, 0, 0, 0, iterations);
       alglib.mincgsetxrep(state, true);
+      //alglib.mincgsetgradientcheck(state, 0.01);
       int neighborSampleSize = neighborSamples;
       Optimize(state, scaledData, classes, dimensions, neighborSampleSize, cancellation, reporter);
+      Optimize(state, scaledData, classes, dimensions, neighborSampleSize, regularization, cancellation, reporter);
       alglib.mincgresults(state, out matrix, out rep);
+      if (rep.terminationtype == -7) throw new InvalidOperationException("Gradient verification failed.");
       var transformationMatrix = new double[attributes, dimensions];
 …
+    }
     private static void Optimize(alglib.mincgstate state, double[,] data, double[] classes, int dimensions, int neighborSampleSize, CancellationToken cancellation, Reporter reporter) {
+    private static void Optimize(alglib.mincgstate state, double[,] data, double[] classes, int dimensions, int neighborSampleSize, double lambda, CancellationToken cancellation, Reporter reporter) {
       while (alglib.mincgiteration(state)) {
         if (cancellation.IsCancellationRequested) break;
         if (state.needfg) {
           Gradient(state.x, ref state.innerobj.f, state.innerobj.g, data, classes, dimensions, neighborSampleSize);
+          Gradient(state.x, ref state.innerobj.f, state.innerobj.g, data, classes, dimensions, neighborSampleSize, lambda);
           continue;
+        }
         if (state.innerobj.xupdated) {
           if (reporter != null)
             reporter(state.innerobj.f, state.innerobj.x);
+            reporter(state.innerobj.f, state.innerobj.x, state.innerobj.g);
           continue;
+        }
 …
+    }
     private static void Gradient(double[] A, ref double func, double[] grad, double[,] data, double[] classes, int dimensions, int neighborSampleSize) {
+    private static void Gradient(double[] A, ref double func, double[] grad, double[,] data, double[] classes, int dimensions, int neighborSampleSize, double lambda) {
       var instances = data.GetLength(0);
       var attributes = data.GetLength(1);
 …
+          }
           var kVector = new Matrix(GetRow(data, k));
+          transformedDistances[k] = Math.Exp(-iVector.Multiply(AMatrix).Subtract(kVector.Multiply(AMatrix)).SquaredVectorLength());
+        }
+        var sample = transformedDistances.OrderByDescending(x => x.Value).Take(neighborSampleSize).ToArray();
+        var normalization = sample.Sum(x => x.Value);
+        if (normalization > 0) {
+          foreach (var s in sample) {
+            if (s.Value <= 0) break;
+            alglib.sparseset(probabilities, i, s.Key, s.Value / normalization);
+          }
+          transformedDistances[k] = Math.Exp(-iVector.Multiply(AMatrix).Subtract(kVector.Multiply(AMatrix)).SumOfSquares());
+        }
+        var normalization = transformedDistances.Sum(x => x.Value);
+        if (normalization <= 0) continue;
+        foreach (var s in transformedDistances.Where(x => x.Value > 0).OrderByDescending(x => x.Value).Take(neighborSampleSize)) {
+          alglib.sparseset(probabilities, i, s.Key, s.Value / normalization);
+        }
+      }
 …
+      }
       func = -pi.Sum();
+      func = -pi.Sum() + lambda * AMatrix.SumOfSquares();
       r = 0;
       var newGrad = AMatrix.Multiply(-2.0).Transpose().Multiply(new Matrix(innerSum)).Transpose();
       foreach (var g in newGrad) {
+        grad[r++] = g;
+      }
+    }
+    private void ReportQuality(double func, double[] coefficients) {
+        grad[r] = g + lambda * 2 * A[r];
+        r++;
+      }
+    }
+    private void ReportQuality(double func, double[] coefficients, double[] gradients) {
       var instances = Problem.ProblemData.TrainingIndices.Count();
       DataTable qualities;
 …
       qualities.Rows["Quality"].Values.Add(-func / instances);
+      string[] attributNames = Problem.ProblemData.AllowedInputVariables.ToArray();
+      if (gradients != null) {
+        DataTable grads;
+        if (!Results.ContainsKey("Gradients")) {
+          grads = new DataTable("Gradients");
+          for (int i = 0; i < gradients.Length; i++)
+            grads.Rows.Add(new DataRow(attributNames[i / Dimensions] + "-" + (i % Dimensions), string.Empty));
+          Results.Add(new Result("Gradients", grads));
+        } else grads = (DataTable)Results["Gradients"].Value;
+        for (int i = 0; i < gradients.Length; i++)
+          grads.Rows[attributNames[i / Dimensions] + "-" + (i % Dimensions)].Values.Add(gradients[i]);
+      }
       if (!Results.ContainsKey("Quality")) {
         Results.Add(new Result("Quality", new DoubleValue(-func / instances)));
       } else ((DoubleValue)Results["Quality"].Value).Value = -func / instances;
+      var attributes = attributNames.Length;
+      var transformationMatrix = new double[attributes, Dimensions];
+      var counter = 0;
+      for (var i = 0; i < attributes; i++)
+        for (var j = 0; j < Dimensions; j++)
+          transformationMatrix[i, j] = coefficients[counter++];
+      var scaling = new Scaling(Problem.ProblemData.Dataset, attributNames, Problem.ProblemData.TrainingIndices);
+      var model = new NcaModel(K, transformationMatrix, Problem.ProblemData.Dataset, Problem.ProblemData.TrainingIndices, Problem.ProblemData.TargetVariable, attributNames, scaling, Problem.ProblemData.ClassValues.ToArray());
+      IClassificationSolution solution = model.CreateClassificationSolution(Problem.ProblemData);
+      if (!Results.ContainsKey("ClassificationSolution")) {
+        Results.Add(new Result("ClassificationSolution", solution));
+      } else {
+        Results["ClassificationSolution"].Value = solution;
+      }
+    }
 …
         yield return data[row, i];
+    }
+  }
+}

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaClassificationSolution.cs

r8471	r8742
42	42	public NcaClassificationSolution(IClassificationProblemData problemData, INcaModel ncaModel)
43	43	: base(ncaModel, problemData) {
44		~~RecalculateResults();~~
45	44	}
46	45
…	…
48	47	return new NcaClassificationSolution(this, cloner);
49	48	}
50
51		~~protected override void RecalculateResults() {~~
52		~~CalculateResults();~~
53		}
54	49	}
55	50	}

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs

r8471	r8742
82	82
83	83	public INcaClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
84		return new NcaClassificationSolution(~~problemData~~, this);
	84	return new NcaClassificationSolution(new ClassificationProblemData(problemData), this);
85	85	}
86	86

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassificationSolution.cs

r7259	r8742
45	45	public NearestNeighbourClassificationSolution(IClassificationProblemData problemData, INearestNeighbourModel nnModel)
46	46	: base(nnModel, problemData) {
47		~~RecalculateResults();~~
48	47	}
49	48
…	…
51	50	return new NearestNeighbourClassificationSolution(this, cloner);
52	51	}
53
54		~~protected override void RecalculateResults() {~~
55		~~CalculateResults();~~
56		}
57	52	}
58	53	}

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs

r8477	r8742
202	202
203	203	public INearestNeighbourRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
204		return new NearestNeighbourRegressionSolution(~~problemData~~, this);
	204	return new NearestNeighbourRegressionSolution(new RegressionProblemData(problemData), this);
205	205	}
206	206	IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
…	…
208	208	}
209	209	public INearestNeighbourClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
210		return new NearestNeighbourClassificationSolution(~~problemData~~, this);
	210	return new NearestNeighbourClassificationSolution(new ClassificationProblemData(problemData), this);
211	211	}
212	212	IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassificationSolution.cs

r7259	r8742
45	45	public NeuralNetworkClassificationSolution(IClassificationProblemData problemData, INeuralNetworkModel nnModel)
46	46	: base(nnModel, problemData) {
47		~~RecalculateResults();~~
48	47	}
49	48
…	…
51	50	return new NeuralNetworkClassificationSolution(this, cloner);
52	51	}
53		protected override void RecalculateResults() {
54		CalculateResults();
55		}
	52
56	53	}
57	54	}

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassificationSolution.cs

r7259	r8742
45	45	public NeuralNetworkEnsembleClassificationSolution(IClassificationProblemData problemData, INeuralNetworkEnsembleModel nnModel)
46	46	: base(nnModel, problemData) {
47		~~RecalculateResults();~~
48	47	}
49	48
…	…
51	50	return new NeuralNetworkEnsembleClassificationSolution(this, cloner);
52	51	}
53
54		~~protected override void RecalculateResults() {~~
55		~~CalculateResults();~~
56		}
57	52	}
58	53	}

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleModel.cs

r7694	r8742
130	130
131	131	public INeuralNetworkEnsembleRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
132		return new NeuralNetworkEnsembleRegressionSolution(~~problemData~~, this);
	132	return new NeuralNetworkEnsembleRegressionSolution(new RegressionEnsembleProblemData(problemData), this);
133	133	}
134	134	IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
…	…
136	136	}
137	137	public INeuralNetworkEnsembleClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
138		return new NeuralNetworkEnsembleClassificationSolution(~~problemData~~, this);
	138	return new NeuralNetworkEnsembleClassificationSolution(new ClassificationEnsembleProblemData(problemData), this);
139	139	}
140	140	IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkModel.cs

r7259	r8742
138	138
139	139	public INeuralNetworkRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
140		return new NeuralNetworkRegressionSolution(~~problemData~~, this);
	140	return new NeuralNetworkRegressionSolution(new RegressionProblemData(problemData), this);
141	141	}
142	142	IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
…	…
144	144	}
145	145	public INeuralNetworkClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
146		return new NeuralNetworkClassificationSolution(~~problemData~~, this);
	146	return new NeuralNetworkClassificationSolution(new ClassificationProblemData(problemData), this);
147	147	}
148	148	IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame

r8477	r8742
46	46	[PluginDependency("HeuristicLab.Problems.DataAnalysis.Symbolic.Classification", "3.4")]
47	47	[PluginDependency("HeuristicLab.Problems.DataAnalysis.Symbolic.Regression", "3.4")]
48		[PluginDependency("HeuristicLab.LibSVM", "~~1.6.3~~")]
	48	[PluginDependency("HeuristicLab.LibSVM", "3.12")]
49	49	[PluginDependency("HeuristicLab.Random", "3.3")]
50	50	public class HeuristicLabAlgorithmsDataAnalysisPlugin : PluginBase {

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassificationSolution.cs

r7259	r8742
45	45	public RandomForestClassificationSolution(IClassificationProblemData problemData, IRandomForestModel randomForestModel)
46	46	: base(randomForestModel, problemData) {
47		~~RecalculateResults();~~
48	47	}
49	48
…	…
51	50	return new RandomForestClassificationSolution(this, cloner);
52	51	}
53
54		~~protected override void RecalculateResults() {~~
55		~~CalculateResults();~~
56		}
57	52	}
58	53	}

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs

r7259	r8742
132	132
133	133	public IRandomForestRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
134		return new RandomForestRegressionSolution(~~problemData~~, this);
	134	return new RandomForestRegressionSolution(new RegressionProblemData(problemData), this);
135	135	}
136	136	IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
…	…
138	138	}
139	139	public IRandomForestClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
140		return new RandomForestClassificationSolution(~~problemData~~, this);
	140	return new RandomForestClassificationSolution(new ClassificationProblemData(problemData), this);
141	141	}
142	142	IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassification.cs

-                      r8430
+                      r8742
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
+using LibSVM;
 namespace HeuristicLab.Algorithms.DataAnalysis {
 …
     private const string NuParameterName = "Nu";
     private const string GammaParameterName = "Gamma";
+    private const string DegreeParameterName = "Degree";
     #region parameter properties
 …
     public IValueParameter<DoubleValue> GammaParameter {
       get { return (IValueParameter<DoubleValue>)Parameters[GammaParameterName]; }
+    }
+    public IValueParameter<IntValue> DegreeParameter {
+      get { return (IValueParameter<IntValue>)Parameters[DegreeParameterName]; }
+    }
     #endregion
 …
     public DoubleValue Gamma {
       get { return GammaParameter.Value; }
+    }
+    public IntValue Degree {
+      get { return DegreeParameter.Value; }
+    }
     #endregion
 …
       Parameters.Add(new ValueParameter<DoubleValue>(CostParameterName, "The value of the C (cost) parameter of C-SVC.", new DoubleValue(1.0)));
       Parameters.Add(new ValueParameter<DoubleValue>(GammaParameterName, "The value of the gamma parameter in the kernel function.", new DoubleValue(1.0)));
+      Parameters.Add(new ValueParameter<IntValue>(DegreeParameterName, "The degree parameter for the polynomial kernel function.", new IntValue(3)));
+    }
     [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+    private void AfterDeserialization() {
+      #region backwards compatibility (change with 3.4)
+      if (!Parameters.ContainsKey(DegreeParameterName))
+        Parameters.Add(new ValueParameter<IntValue>(DegreeParameterName, "The degree parameter for the polynomial kernel function.", new IntValue(3)));
+      #endregion
+    }
     public override IDeepCloneable Clone(Cloner cloner) {
 …
       int nSv;
       var solution = CreateSupportVectorClassificationSolution(problemData, selectedInputVariables,
         SvmType.Value, KernelType.Value, Cost.Value, Nu.Value, Gamma.Value,
+        SvmType.Value, KernelType.Value, Cost.Value, Nu.Value, Gamma.Value, Degree.Value,
         out trainingAccuracy, out testAccuracy, out nSv);
       Results.Add(new Result("Support vector classification solution", "The support vector classification solution.", solution));
       Results.Add(new Result("Training accuracy", "The accuracy of the SVR solution on the training partition.", new DoubleValue(trainingAccuracy)));
       Results.Add(new Result("Test R²", "The accuracy of the SVR solution on the test partition.", new DoubleValue(testAccuracy)));
+      Results.Add(new Result("Test accuracy", "The accuracy of the SVR solution on the test partition.", new DoubleValue(testAccuracy)));
       Results.Add(new Result("Number of support vectors", "The number of support vectors of the SVR solution.", new IntValue(nSv)));
+    }
     public static SupportVectorClassificationSolution CreateSupportVectorClassificationSolution(IClassificationProblemData problemData, IEnumerable<string> allowedInputVariables,
       string svmType, string kernelType, double cost, double nu, double gamma,
+      string svmType, string kernelType, double cost, double nu, double gamma, int degree,
       out double trainingAccuracy, out double testAccuracy, out int nSv) {
       Dataset dataset = problemData.Dataset;
 …
       //extract SVM parameters from scope and set them
       SVM.Parameter parameter = new SVM.Parameter();
       parameter.SvmType = (SVM.SvmType)Enum.Parse(typeof(SVM.SvmType), svmType, true);
       parameter.KernelType = (SVM.KernelType)Enum.Parse(typeof(SVM.KernelType), kernelType, true);
+      svm_parameter parameter = new svm_parameter();
+      parameter.svm_type = GetSvmType(svmType);
+      parameter.kernel_type = GetKernelType(kernelType);
       parameter.C = cost;
+      parameter.Nu = nu;
+      parameter.Gamma = gamma;
+      parameter.CacheSize = 500;
+      parameter.Probability = false;
+      parameter.nu = nu;
+      parameter.gamma = gamma;
+      parameter.cache_size = 500;
+      parameter.probability = 0;
+      parameter.eps = 0.001;
+      parameter.degree = degree;
+      parameter.shrinking = 1;
+      parameter.coef0 = 0;
+      var weightLabels = new List<int>();
+      var weights = new List<double>();
       foreach (double c in problemData.ClassValues) {
         double wSum = 0.0;
 …
+          }
+        }
+        parameter.Weights.Add((int)c, wSum);
+        weightLabels.Add((int)c);
+        weights.Add(wSum);
+      }
+      SVM.Problem problem = SupportVectorMachineUtil.CreateSvmProblem(dataset, targetVariable, allowedInputVariables, rows);
+      SVM.RangeTransform rangeTransform = SVM.RangeTransform.Compute(problem);
+      SVM.Problem scaledProblem = SVM.Scaling.Scale(rangeTransform, problem);
+      var svmModel = SVM.Training.Train(scaledProblem, parameter);
+      parameter.weight_label = weightLabels.ToArray();
+      parameter.weight = weights.ToArray();
+      svm_problem problem = SupportVectorMachineUtil.CreateSvmProblem(dataset, targetVariable, allowedInputVariables, rows);
+      RangeTransform rangeTransform = RangeTransform.Compute(problem);
+      svm_problem scaledProblem = rangeTransform.Scale(problem);
+      var svmModel = svm.svm_train(scaledProblem, parameter);
       var model = new SupportVectorMachineModel(svmModel, rangeTransform, targetVariable, allowedInputVariables, problemData.ClassValues);
       var solution = new SupportVectorClassificationSolution(model, (IClassificationProblemData)problemData.Clone());
       nSv = svmModel.SupportVectorCount;
+      nSv = svmModel.SV.Length;
       trainingAccuracy = solution.TrainingAccuracy;
       testAccuracy = solution.TestAccuracy;
       return solution;
+    }
+    private static int GetSvmType(string svmType) {
+      if (svmType == "NU_SVC") return svm_parameter.NU_SVC;
+      if (svmType == "C_SVC") return svm_parameter.C_SVC;
+      throw new ArgumentException("Unknown SVM type");
+    }
+    private static int GetKernelType(string kernelType) {
+      if (kernelType == "LINEAR") return svm_parameter.LINEAR;
+      if (kernelType == "POLY") return svm_parameter.POLY;
+      if (kernelType == "SIGMOID") return svm_parameter.SIGMOID;
+      if (kernelType == "RBF") return svm_parameter.RBF;
+      throw new ArgumentException("Unknown kernel type");
+    }
     #endregion

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassificationSolution.cs

r7259	r8742
45	45	public SupportVectorClassificationSolution(SupportVectorMachineModel model, IClassificationProblemData problemData)
46	46	: base(model, problemData) {
47		~~RecalculateResults();~~
48	47	}
49	48
…	…
51	50	return new SupportVectorClassificationSolution(this, cloner);
52	51	}
53
54		~~protected override void RecalculateResults() {~~
55		~~CalculateResults();~~
56		}
57	52	}
58	53	}

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineModel.cs

-                      r8430
+                      r8742
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
 using SVM;
+using LibSVM;
 namespace HeuristicLab.Algorithms.DataAnalysis {
 …
   public sealed class SupportVectorMachineModel : NamedItem, ISupportVectorMachineModel {
     private SVM.Model model;
+    private svm_model model;
     /// <summary>
     /// Gets or sets the SVM model.
     /// </summary>
     public SVM.Model Model {
+    public svm_model Model {
       get { return model; }
       set {
 …
     /// Gets or sets the range transformation for the model.
     /// </summary>
     private SVM.RangeTransform rangeTransform;
     public SVM.RangeTransform RangeTransform {
+    private RangeTransform rangeTransform;
+    public RangeTransform RangeTransform {
       get { return rangeTransform; }
       set {
 …
     public Dataset SupportVectors {
       get {
         var data = new double[Model.SupportVectorCount, allowedInputVariables.Count()];
         for (int i = 0; i < Model.SupportVectorCount; i++) {
           var sv = Model.SupportVectors[i];
+        var data = new double[Model.sv_coef.Length, allowedInputVariables.Count()];
+        for (int i = 0; i < Model.sv_coef.Length; i++) {
+          var sv = Model.SV[i];
           for (int j = 0; j < sv.Length; j++) {
             data[i, j] = sv[j].Value;
+            data[i, j] = sv[j].value;
+          }
+        }
 …
         this.classValues = (double[])original.classValues.Clone();
+    }
     public SupportVectorMachineModel(SVM.Model model, SVM.RangeTransform rangeTransform, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<double> classValues)
+    public SupportVectorMachineModel(svm_model model, RangeTransform rangeTransform, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<double> classValues)
       : this(model, rangeTransform, targetVariable, allowedInputVariables) {
       this.classValues = classValues.ToArray();
+    }
     public SupportVectorMachineModel(SVM.Model model, SVM.RangeTransform rangeTransform, string targetVariable, IEnumerable<string> allowedInputVariables)
+    public SupportVectorMachineModel(svm_model model, RangeTransform rangeTransform, string targetVariable, IEnumerable<string> allowedInputVariables)
       : base() {
       this.name = ItemName;
 …
+    }
     public SupportVectorRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
       return new SupportVectorRegressionSolution(this, problemData);
+      return new SupportVectorRegressionSolution(this, new RegressionProblemData(problemData));
+    }
     IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
 …
     public SupportVectorClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
       return new SupportVectorClassificationSolution(this, problemData);
+      return new SupportVectorClassificationSolution(this, new ClassificationProblemData(problemData));
+    }
     IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {
 …
     private IEnumerable<double> GetEstimatedValuesHelper(Dataset dataset, IEnumerable<int> rows) {
       // calculate predictions for the currently requested rows
       SVM.Problem problem = SupportVectorMachineUtil.CreateSvmProblem(dataset, targetVariable, allowedInputVariables, rows);
       SVM.Problem scaledProblem = SVM.Scaling.Scale(RangeTransform, problem);
       for (int i = 0; i < scaledProblem.Count; i++) {
         yield return SVM.Prediction.Predict(Model, scaledProblem.X[i]);
+      svm_problem problem = SupportVectorMachineUtil.CreateSvmProblem(dataset, targetVariable, allowedInputVariables, rows);
+      svm_problem scaledProblem = rangeTransform.Scale(problem);
+      for (int i = 0; i < problem.l; i++) {
+        yield return svm.svm_predict(Model, scaledProblem.x[i]);
+      }
+    }
 …
       get {
         using (MemoryStream stream = new MemoryStream()) {
           SVM.Model.Write(stream, Model);
+          svm.svm_save_model(new StreamWriter(stream), Model);
           stream.Seek(0, System.IO.SeekOrigin.Begin);
           StreamReader reader = new StreamReader(stream);
 …
       set {
         using (MemoryStream stream = new MemoryStream(Encoding.ASCII.GetBytes(value))) {
           model = SVM.Model.Read(stream);
+          model = svm.svm_load_model(new StreamReader(stream));
+        }
+      }
 …
       get {
         using (MemoryStream stream = new MemoryStream()) {
           SVM.RangeTransform.Write(stream, RangeTransform);
+          RangeTransform.Write(stream, RangeTransform);
           stream.Seek(0, System.IO.SeekOrigin.Begin);
           StreamReader reader = new StreamReader(stream);
 …
       set {
         using (MemoryStream stream = new MemoryStream(Encoding.ASCII.GetBytes(value))) {
           RangeTransform = SVM.RangeTransform.Read(stream);
+          RangeTransform = RangeTransform.Read(stream);
+        }
+      }

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineUtil.cs

-                      r7259
+                      r8742
 using System.Linq;
 using HeuristicLab.Problems.DataAnalysis;
+using LibSVM;
 namespace HeuristicLab.Algorithms.DataAnalysis {
 …
     /// <param name="rowIndices">The rows of the dataset that should be contained in the resulting SVM-problem</param>
     /// <returns>A problem data type that can be used to train a support vector machine.</returns>
     public static SVM.Problem CreateSvmProblem(Dataset dataset, string targetVariable, IEnumerable<string> inputVariables, IEnumerable<int> rowIndices) {
+    public static svm_problem CreateSvmProblem(Dataset dataset, string targetVariable, IEnumerable<string> inputVariables, IEnumerable<int> rowIndices) {
       double[] targetVector =
         dataset.GetDoubleValues(targetVariable, rowIndices).ToArray();
       SVM.Node[][] nodes = new SVM.Node[targetVector.Length][];
       List<SVM.Node> tempRow;
+      svm_node[][] nodes = new svm_node[targetVector.Length][];
+      List<svm_node> tempRow;
       int maxNodeIndex = 0;
       int svmProblemRowIndex = 0;
       List<string> inputVariablesList = inputVariables.ToList();
       foreach (int row in rowIndices) {
         tempRow = new List<SVM.Node>();
+        tempRow = new List<svm_node>();
         int colIndex = 1; // make sure the smallest node index for SVM = 1
         foreach (var inputVariable in inputVariablesList) {
 …
           // => don't add NaN values in the dataset to the sparse SVM matrix representation
           if (!double.IsNaN(value)) {
             tempRow.Add(new SVM.Node(colIndex, value)); // nodes must be sorted in ascending ordered by column index
+            tempRow.Add(new svm_node() { index = colIndex, value = value }); // nodes must be sorted in ascending ordered by column index
             if (colIndex > maxNodeIndex) maxNodeIndex = colIndex;
+          }
 …
+      }
       return new SVM.Problem(targetVector.Length, targetVector, nodes, maxNodeIndex);
+      return new svm_problem() { l = targetVector.Length, y = targetVector, x = nodes };
+    }
+  }

branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegression.cs

-                      r8430
+                      r8742
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
+using LibSVM;
 namespace HeuristicLab.Algorithms.DataAnalysis {
 …
     private const string GammaParameterName = "Gamma";
     private const string EpsilonParameterName = "Epsilon";
+    private const string DegreeParameterName = "Degree";
     #region parameter properties
 …
     public IValueParameter<DoubleValue> EpsilonParameter {
       get { return (IValueParameter<DoubleValue>)Parameters[EpsilonParameterName]; }
+    }
+    public IValueParameter<IntValue> DegreeParameter {
+      get { return (IValueParameter<IntValue>)Parameters[DegreeParameterName]; }
+    }
     #endregion
 …
     public DoubleValue Epsilon {
       get { return EpsilonParameter.Value; }
+    }
+    public IntValue Degree {
+      get { return DegreeParameter.Value; }
+    }
     #endregion
 …
       Parameters.Add(new ValueParameter<DoubleValue>(GammaParameterName, "The value of the gamma parameter in the kernel function.", new DoubleValue(1.0)));
       Parameters.Add(new ValueParameter<DoubleValue>(EpsilonParameterName, "The value of the epsilon parameter for epsilon-SVR.", new DoubleValue(0.1)));
+      Parameters.Add(new ValueParameter<IntValue>(DegreeParameterName, "The degree parameter for the polynomial kernel function.", new IntValue(3)));
+    }
     [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+    private void AfterDeserialization() {
+      #region backwards compatibility (change with 3.4)
+      if (!Parameters.ContainsKey(DegreeParameterName))
+        Parameters.Add(new ValueParameter<IntValue>(DegreeParameterName, "The degree parameter for the polynomial kernel function.", new IntValue(3)));
+      #endregion
+    }
     public override IDeepCloneable Clone(Cloner cloner) {
 …
       int nSv;
       var solution = CreateSupportVectorRegressionSolution(problemData, selectedInputVariables, SvmType.Value,
         KernelType.Value, Cost.Value, Nu.Value, Gamma.Value, Epsilon.Value,
+        KernelType.Value, Cost.Value, Nu.Value, Gamma.Value, Epsilon.Value, Degree.Value,
         out trainR2, out testR2, out nSv);
 …
     public static SupportVectorRegressionSolution CreateSupportVectorRegressionSolution(IRegressionProblemData problemData, IEnumerable<string> allowedInputVariables,
       string svmType, string kernelType, double cost, double nu, double gamma, double epsilon,
+      string svmType, string kernelType, double cost, double nu, double gamma, double epsilon, int degree,
       out double trainingR2, out double testR2, out int nSv) {
       Dataset dataset = problemData.Dataset;
 …
       //extract SVM parameters from scope and set them
       SVM.Parameter parameter = new SVM.Parameter();
       parameter.SvmType = (SVM.SvmType)Enum.Parse(typeof(SVM.SvmType), svmType, true);
       parameter.KernelType = (SVM.KernelType)Enum.Parse(typeof(SVM.KernelType), kernelType, true);
+      svm_parameter parameter = new svm_parameter();
+      parameter.svm_type = GetSvmType(svmType);
+      parameter.kernel_type = GetKernelType(kernelType);
       parameter.C = cost;
+      parameter.Nu = nu;
+      parameter.Gamma = gamma;
+      parameter.P = epsilon;
+      parameter.CacheSize = 500;
+      parameter.Probability = false;
+      SVM.Problem problem = SupportVectorMachineUtil.CreateSvmProblem(dataset, targetVariable, allowedInputVariables, rows);
+      SVM.RangeTransform rangeTransform = SVM.RangeTransform.Compute(problem);
+      SVM.Problem scaledProblem = SVM.Scaling.Scale(rangeTransform, problem);
+      var svmModel = SVM.Training.Train(scaledProblem, parameter);
+      nSv = svmModel.SupportVectorCount;
+      parameter.nu = nu;
+      parameter.gamma = gamma;
+      parameter.p = epsilon;
+      parameter.cache_size = 500;
+      parameter.probability = 0;
+      parameter.eps = 0.001;
+      parameter.degree = degree;
+      parameter.shrinking = 1;
+      parameter.coef0 = 0;
+      svm_problem problem = SupportVectorMachineUtil.CreateSvmProblem(dataset, targetVariable, allowedInputVariables, rows);
+      RangeTransform rangeTransform = RangeTransform.Compute(problem);
+      svm_problem scaledProblem = rangeTransform.Scale(problem);
+      var svmModel = svm.svm_train(scaledProblem, parameter);
+      nSv = svmModel.SV.Length;
       var model = new SupportVectorMachineModel(svmModel, rangeTransform, targetVariable, allowedInputVariables);
       var solution = new SupportVectorRegressionSolution(model, (IRegressionProblemData)problemData.Clone());
 …
       return solution;
+    }
+    private static int GetSvmType(string svmType) {
+      if (svmType == "NU_SVR") return svm_parameter.NU_SVR;
+      if (svmType == "EPSILON_SVR") return svm_parameter.EPSILON_SVR;
+      throw new ArgumentException("Unknown SVM type");
+    }
+    private static int GetKernelType(string kernelType) {
+      if (kernelType == "LINEAR") return svm_parameter.LINEAR;
+      if (kernelType == "POLY") return svm_parameter.POLY;
+      if (kernelType == "SIGMOID") return svm_parameter.SIGMOID;
+      if (kernelType == "RBF") return svm_parameter.RBF;
+      throw new ArgumentException("Unknown kernel type");
+    }
     #endregion
+  }

Context Navigation

Legend:

Download in other formats: