Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneFactorClassificationModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneFactorClassificationModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneFactorClassificationModel.cs	(revision 17041)
@@ -0,0 +1,103 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("1FD28BA4-E30E-48E8-B868-24A5F2363DD0")]
+  [Item("OneFactor Classification Model", "A model that uses only one categorical feature (factor) to determine the class.")]
+  public sealed class OneFactorClassificationModel : ClassificationModel {
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return new[] { Variable }; }
+    }
+
+    [Storable]
+    private string variable;
+    public string Variable {
+      get { return variable; }
+    }
+
+    [Storable]
+    private string[] variableValues;
+    public string[] VariableValues {
+      get { return variableValues; }
+    }
+
+    [Storable]
+    private double[] classes;
+    public double[] Classes {
+      get { return classes; }
+    }
+
+    [Storable]
+    private double defaultClass;
+    public double DefaultClass {
+      get { return defaultClass; }
+    }
+
+    [StorableConstructor]
+    private OneFactorClassificationModel(StorableConstructorFlag _) : base(_) { }
+    private OneFactorClassificationModel(OneFactorClassificationModel original, Cloner cloner)
+      : base(original, cloner) {
+      this.variable = (string)original.variable;
+      this.variableValues = (string[])original.variableValues.Clone();
+      this.classes = (double[])original.classes.Clone();
+      this.defaultClass = original.defaultClass;
+    }
+    public override IDeepCloneable Clone(Cloner cloner) { return new OneFactorClassificationModel(this, cloner); }
+
+    public OneFactorClassificationModel(string targetVariable, string variable, string[] variableValues, double[] classes, double defaultClass = double.NaN)
+      : base(targetVariable) {
+      if (variableValues.Length != classes.Length) {
+        throw new ArgumentException("Number of variable values and classes has to be equal.");
+      }
+      this.name = ItemName;
+      this.description = ItemDescription;
+      this.variable = variable;
+      this.variableValues = variableValues;
+      this.classes = classes;
+      this.defaultClass = double.IsNaN(defaultClass) ? classes.First() : defaultClass;
+      Array.Sort(variableValues, classes);
+    }
+
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+      return dataset.GetStringValues(Variable, rows)
+        .Select(GetPredictedValueForInput);
+    }
+
+    private double GetPredictedValueForInput(string val) {
+      var matchingIdx = Array.BinarySearch(variableValues, val);
+      if (matchingIdx >= 0) return classes[matchingIdx];
+      else return DefaultClass;
+    }
+
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new OneFactorClassificationSolution(this, new ClassificationProblemData(problemData));
+    }
+
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneFactorClassificationSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneFactorClassificationSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneFactorClassificationSolution.cs	(revision 17041)
@@ -0,0 +1,48 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("7A180AA0-E963-4714-A33A-2C87C4108B93")]
+  [Item(Name = "OneR Classification Solution", Description = "Represents a OneR classification solution which uses only a single factor for class prediction.")]
+  public sealed class OneFactorClassificationSolution : ClassificationSolution {
+    public new OneFactorClassificationModel Model {
+      get { return (OneFactorClassificationModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private OneFactorClassificationSolution(StorableConstructorFlag _) : base(_) { }
+    private OneFactorClassificationSolution(OneFactorClassificationSolution original, Cloner cloner) : base(original, cloner) { }
+    public OneFactorClassificationSolution(OneFactorClassificationModel model, IClassificationProblemData problemData)
+      : base(model, problemData) {
+      RecalculateResults();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new OneFactorClassificationSolution(this, cloner);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneR.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneR.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneR.cs	(revision 17041)
@@ -0,0 +1,238 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// 1R classification algorithm.
+  /// </summary>
+  [Item("OneR Classification", "A simple classification algorithm the searches the best single-variable split (does not support categorical features correctly). See R.C. Holte (1993). Very simple classification rules perform well on most commonly used datasets. Machine Learning. 11:63-91.")]
+  [StorableType("22D1C518-CEDA-413C-8997-D34BC06B6267")]
+  public sealed class OneR : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+
+    public IValueParameter<IntValue> MinBucketSizeParameter {
+      get { return (IValueParameter<IntValue>)Parameters["MinBucketSize"]; }
+    }
+
+    [StorableConstructor]
+    private OneR(StorableConstructorFlag _) : base(_) { }
+
+    private OneR(OneR original, Cloner cloner)
+      : base(original, cloner) { }
+
+    public OneR()
+      : base() {
+      Parameters.Add(new ValueParameter<IntValue>("MinBucketSize", "Minimum size of a bucket for numerical values. (Except for the rightmost bucket)", new IntValue(6)));
+      Problem = new ClassificationProblem();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new OneR(this, cloner);
+    }
+
+    protected override void Run(CancellationToken cancellationToken) {
+      var solution = CreateOneRSolution(Problem.ProblemData, MinBucketSizeParameter.Value.Value);
+      Results.Add(new Result("OneR solution", "The 1R classifier.", solution));
+    }
+
+    public static IClassificationSolution CreateOneRSolution(IClassificationProblemData problemData, int minBucketSize = 6) {
+      var classValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+      var model1 = FindBestDoubleVariableModel(problemData, minBucketSize);
+      var model2 = FindBestFactorModel(problemData);
+
+      if (model1 == null && model2 == null) throw new InvalidProgramException("Could not create OneR solution");
+      else if (model1 == null) return new OneFactorClassificationSolution(model2, (IClassificationProblemData)problemData.Clone());
+      else if (model2 == null) return new OneRClassificationSolution(model1, (IClassificationProblemData)problemData.Clone());
+      else {
+        var model1EstimatedValues = model1.GetEstimatedClassValues(problemData.Dataset, problemData.TrainingIndices);
+        var model1NumCorrect = classValues.Zip(model1EstimatedValues, (a, b) => a.IsAlmost(b)).Count(e => e);
+
+        var model2EstimatedValues = model2.GetEstimatedClassValues(problemData.Dataset, problemData.TrainingIndices);
+        var model2NumCorrect = classValues.Zip(model2EstimatedValues, (a, b) => a.IsAlmost(b)).Count(e => e);
+
+        if (model1NumCorrect > model2NumCorrect) {
+          return new OneRClassificationSolution(model1, (IClassificationProblemData)problemData.Clone());
+        } else {
+          return new OneFactorClassificationSolution(model2, (IClassificationProblemData)problemData.Clone());
+        }
+      }
+    }
+
+    private static OneRClassificationModel FindBestDoubleVariableModel(IClassificationProblemData problemData, int minBucketSize = 6) {
+      var bestClassified = 0;
+      List<Split> bestSplits = null;
+      string bestVariable = string.Empty;
+      double bestMissingValuesClass = double.NaN;
+      var classValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+
+      var allowedInputVariables = problemData.AllowedInputVariables.Where(problemData.Dataset.VariableHasType<double>);
+
+      if (!allowedInputVariables.Any()) return null;
+
+      foreach (var variable in allowedInputVariables) {
+        var inputValues = problemData.Dataset.GetDoubleValues(variable, problemData.TrainingIndices);
+        var samples = inputValues.Zip(classValues, (i, v) => new Sample(i, v)).OrderBy(s => s.inputValue);
+
+        var missingValuesDistribution = samples
+          .Where(s => double.IsNaN(s.inputValue)).GroupBy(s => s.classValue)
+          .ToDictionary(s => s.Key, s => s.Count())
+          .MaxItems(s => s.Value)
+          .FirstOrDefault();
+
+        //calculate class distributions for all distinct inputValues
+        List<Dictionary<double, int>> classDistributions = new List<Dictionary<double, int>>();
+        List<double> thresholds = new List<double>();
+        double lastValue = double.NaN;
+        foreach (var sample in samples.Where(s => !double.IsNaN(s.inputValue))) {
+          if (sample.inputValue > lastValue || double.IsNaN(lastValue)) {
+            if (!double.IsNaN(lastValue)) thresholds.Add((lastValue + sample.inputValue) / 2);
+            lastValue = sample.inputValue;
+            classDistributions.Add(new Dictionary<double, int>());
+            foreach (var classValue in problemData.ClassValues)
+              classDistributions[classDistributions.Count - 1][classValue] = 0;
+
+          }
+          classDistributions[classDistributions.Count - 1][sample.classValue]++;
+        }
+        thresholds.Add(double.PositiveInfinity);
+
+        var distribution = classDistributions[0];
+        var threshold = thresholds[0];
+        var splits = new List<Split>();
+
+        for (int i = 1; i < classDistributions.Count; i++) {
+          var samplesInSplit = distribution.Max(d => d.Value);
+          //join splits if there are too few samples in the split or the distributions has the same maximum class value as the current split
+          if (samplesInSplit < minBucketSize ||
+            classDistributions[i].MaxItems(d => d.Value).Select(d => d.Key).Contains(
+              distribution.MaxItems(d => d.Value).Select(d => d.Key).First())) {
+            foreach (var classValue in classDistributions[i])
+              distribution[classValue.Key] += classValue.Value;
+            threshold = thresholds[i];
+          } else {
+            splits.Add(new Split(threshold, distribution.MaxItems(d => d.Value).Select(d => d.Key).First()));
+            distribution = classDistributions[i];
+            threshold = thresholds[i];
+          }
+        }
+        splits.Add(new Split(double.PositiveInfinity, distribution.MaxItems(d => d.Value).Select(d => d.Key).First()));
+
+        int correctClassified = 0;
+        int splitIndex = 0;
+        foreach (var sample in samples.Where(s => !double.IsNaN(s.inputValue))) {
+          while (sample.inputValue >= splits[splitIndex].thresholdValue)
+            splitIndex++;
+          correctClassified += sample.classValue.IsAlmost(splits[splitIndex].classValue) ? 1 : 0;
+        }
+        correctClassified += missingValuesDistribution.Value;
+
+        if (correctClassified > bestClassified) {
+          bestClassified = correctClassified;
+          bestSplits = splits;
+          bestVariable = variable;
+          bestMissingValuesClass = missingValuesDistribution.Value == 0 ? double.NaN : missingValuesDistribution.Key;
+        }
+      }
+
+      //remove neighboring splits with the same class value
+      for (int i = 0; i < bestSplits.Count - 1; i++) {
+        if (bestSplits[i].classValue.IsAlmost(bestSplits[i + 1].classValue)) {
+          bestSplits.Remove(bestSplits[i]);
+          i--;
+        }
+      }
+
+      var model = new OneRClassificationModel(problemData.TargetVariable, bestVariable,
+        bestSplits.Select(s => s.thresholdValue).ToArray(),
+        bestSplits.Select(s => s.classValue).ToArray(), bestMissingValuesClass);
+
+      return model;
+    }
+    private static OneFactorClassificationModel FindBestFactorModel(IClassificationProblemData problemData) {
+      var classValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+      var defaultClass = FindMostFrequentClassValue(classValues);
+      // only select string variables
+      var allowedInputVariables = problemData.AllowedInputVariables.Where(problemData.Dataset.VariableHasType<string>);
+
+      if (!allowedInputVariables.Any()) return null;
+
+      OneFactorClassificationModel bestModel = null;
+      var bestModelNumCorrect = 0;
+
+      foreach (var variable in allowedInputVariables) {
+        var variableValues = problemData.Dataset.GetStringValues(variable, problemData.TrainingIndices);
+        var groupedClassValues = variableValues
+          .Zip(classValues, (v, c) => new KeyValuePair<string, double>(v, c))
+          .GroupBy(kvp => kvp.Key)
+          .ToDictionary(g => g.Key, g => FindMostFrequentClassValue(g.Select(kvp => kvp.Value)));
+
+        var model = new OneFactorClassificationModel(problemData.TargetVariable, variable,
+          groupedClassValues.Select(kvp => kvp.Key).ToArray(), groupedClassValues.Select(kvp => kvp.Value).ToArray(), defaultClass);
+
+        var modelEstimatedValues = model.GetEstimatedClassValues(problemData.Dataset, problemData.TrainingIndices);
+        var modelNumCorrect = classValues.Zip(modelEstimatedValues, (a, b) => a.IsAlmost(b)).Count(e => e);
+        if (modelNumCorrect > bestModelNumCorrect) {
+          bestModelNumCorrect = modelNumCorrect;
+          bestModel = model;
+        }
+      }
+
+      return bestModel;
+    }
+
+    private static double FindMostFrequentClassValue(IEnumerable<double> classValues) {
+      return classValues.GroupBy(c => c).OrderByDescending(g => g.Count()).Select(g => g.Key).First();
+    }
+
+    #region helper classes
+    private class Split {
+      public double thresholdValue;
+      public double classValue;
+
+      public Split(double thresholdValue, double classValue) {
+        this.thresholdValue = thresholdValue;
+        this.classValue = classValue;
+      }
+    }
+
+    private class Sample {
+      public double inputValue;
+      public double classValue;
+
+      public Sample(double inputValue, double classValue) {
+        this.inputValue = inputValue;
+        this.classValue = classValue;
+      }
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationModel.cs	(revision 17041)
@@ -0,0 +1,120 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("C3365B2F-75D6-45F7-9DD9-CD80854F9D75")]
+  [Item("OneR Classification Model", "A model that uses intervals for one variable to determine the class.")]
+  public sealed class OneRClassificationModel : ClassificationModel {
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return new[] { Variable }; }
+    }
+
+    [Storable]
+    private string variable;
+    public string Variable {
+      get { return variable; }
+    }
+
+    [Storable]
+    private double[] splits;
+    public double[] Splits {
+      get { return splits; }
+    }
+
+    [Storable]
+    private double[] classes;
+    public double[] Classes {
+      get { return classes; }
+    }
+
+    [Storable]
+    private double missingValuesClass;
+    public double MissingValuesClass {
+      get { return missingValuesClass; }
+    }
+
+    [StorableConstructor]
+    private OneRClassificationModel(StorableConstructorFlag _) : base(_) { }
+    private OneRClassificationModel(OneRClassificationModel original, Cloner cloner)
+      : base(original, cloner) {
+      this.variable = (string)original.variable;
+      this.splits = (double[])original.splits.Clone();
+      this.classes = (double[])original.classes.Clone();
+      this.missingValuesClass = original.missingValuesClass;
+    }
+    public override IDeepCloneable Clone(Cloner cloner) { return new OneRClassificationModel(this, cloner); }
+
+    public OneRClassificationModel(string targetVariable, string variable, double[] splits, double[] classes, double missingValuesClass = double.NaN)
+      : base(targetVariable) {
+      if (splits.Length != classes.Length) {
+        throw new ArgumentException("Number of splits and classes has to be equal.");
+      }
+      if (!Double.IsPositiveInfinity(splits[splits.Length - 1])) {
+        throw new ArgumentException("Last split has to be double.PositiveInfinity, so that all values are covered.");
+      }
+      this.name = ItemName;
+      this.description = ItemDescription;
+      this.variable = variable;
+      this.splits = splits;
+      this.classes = classes;
+      this.missingValuesClass = missingValuesClass;
+    }
+
+    // uses sorting to return the values in the order of rows, instead of using nested for loops
+    // to avoid O(n²) runtime
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+      var values = dataset.GetDoubleValues(Variable, rows).ToArray();
+      var rowsArray = rows.ToArray();
+      var order = Enumerable.Range(0, rowsArray.Length).ToArray();
+      double[] estimated = new double[rowsArray.Length];
+      Array.Sort(rowsArray, order);
+      Array.Sort(values, rowsArray);
+      int curSplit = 0, curIndex = 0;
+      while (curIndex < values.Length && Double.IsNaN(values[curIndex])) {
+        estimated[curIndex] = MissingValuesClass;
+        curIndex++;
+      }
+      while (curSplit < Splits.Length) {
+        while (curIndex < values.Length && Splits[curSplit] > values[curIndex]) {
+          estimated[curIndex] = classes[curSplit];
+          curIndex++;
+        }
+        curSplit++;
+      }
+      Array.Sort(rowsArray, estimated);
+      Array.Sort(order, estimated);
+      return estimated;
+    }
+
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new OneRClassificationSolution(this, new ClassificationProblemData(problemData));
+    }
+
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationSolution.cs	(revision 17041)
@@ -0,0 +1,48 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("971CA431-D3BC-403C-A0FB-0E6F5BA426D7")]
+  [Item(Name = "OneR Classification Solution", Description = "Represents a OneR classification solution which uses only a single feature with potentially multiple thresholds for class prediction.")]
+  public sealed class OneRClassificationSolution : ClassificationSolution {
+    public new OneRClassificationModel Model {
+      get { return (OneRClassificationModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private OneRClassificationSolution(StorableConstructorFlag _) : base(_) { }
+    private OneRClassificationSolution(OneRClassificationSolution original, Cloner cloner) : base(original, cloner) { }
+    public OneRClassificationSolution(OneRClassificationModel model, IClassificationProblemData problemData)
+      : base(model, problemData) {
+      RecalculateResults();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new OneRClassificationSolution(this, cloner);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/ZeroR.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/ZeroR.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/ZeroR.cs	(revision 17041)
@@ -0,0 +1,72 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Optimization;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// 0R classification algorithm.
+  /// </summary>
+  [Item("ZeroR Classification", "The simplest possible classifier, ZeroR always predicts the majority class.")]
+  [StorableType("A2C4BA0A-008B-44EB-B93A-3B53B867F0EA")]
+  public sealed class ZeroR : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+
+    [StorableConstructor]
+    private ZeroR(StorableConstructorFlag _) : base(_) { }
+    private ZeroR(ZeroR original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public ZeroR()
+      : base() {
+      Problem = new ClassificationProblem();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new ZeroR(this, cloner);
+    }
+
+    protected override void Run(CancellationToken cancellationToken) {
+      var solution = CreateZeroRSolution(Problem.ProblemData);
+      Results.Add(new Result("ZeroR solution", "The simplest possible classifier, ZeroR always predicts the majority class.", solution));
+    }
+
+    public static IClassificationSolution CreateZeroRSolution(IClassificationProblemData problemData) {
+      var dataset = problemData.Dataset;
+      string target = problemData.TargetVariable;
+      var targetValues = dataset.GetDoubleValues(target, problemData.TrainingIndices);
+
+
+      // if multiple classes have the same number of observations then simply take the first one
+      var dominantClass = targetValues.GroupBy(x => x).ToDictionary(g => g.Key, g => g.Count())
+        .MaxItems(kvp => kvp.Value).Select(x => x.Key).First();
+
+      var model = new ConstantModel(dominantClass, target);
+      var solution = model.CreateClassificationSolution(problemData);
+      return solution;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs	(revision 17041)
@@ -0,0 +1,812 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using HeuristicLab.Collections;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Random;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("Cross Validation (CV)", "Cross-validation wrapper for data analysis algorithms.")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 100)]
+  [StorableType("1C622121-AE5B-42FD-831C-FCA8F8E0AF8D")]
+  public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent {
+    [Storable]
+    private int seed;
+
+    private SemaphoreSlim availableWorkers; // limits the number of concurrent algorithm executions
+    private ManualResetEventSlim allAlgorithmsFinished; // this indicates that all started algorithms have been paused or stopped
+
+    public CrossValidation()
+      : base() {
+      name = ItemName;
+      description = ItemDescription;
+
+      executionState = ExecutionState.Stopped;
+      runs = new RunCollection { OptimizerName = name };
+      runsCounter = 0;
+
+      algorithm = null;
+      clonedAlgorithms = new ItemCollection<IAlgorithm>();
+      results = new ResultCollection();
+
+      folds = new IntValue(2);
+      numberOfWorkers = new IntValue(1);
+      samplesStart = new IntValue(0);
+      samplesEnd = new IntValue(0);
+      shuffleSamples = new BoolValue(false);
+      storeAlgorithmInEachRun = false;
+
+      RegisterEvents();
+      if (Algorithm != null) RegisterAlgorithmEvents();
+    }
+
+    public string Filename { get; set; }
+
+    #region persistence and cloning
+    [StorableConstructor]
+    private CrossValidation(StorableConstructorFlag _) : base(_) {
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
+      if (shuffleSamples == null) shuffleSamples = new BoolValue(false);
+      #endregion
+
+      RegisterEvents();
+      if (Algorithm != null) RegisterAlgorithmEvents();
+    }
+
+    private CrossValidation(CrossValidation original, Cloner cloner)
+      : base(original, cloner) {
+      executionState = original.executionState;
+      storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
+      runs = cloner.Clone(original.runs);
+      runsCounter = original.runsCounter;
+      algorithm = cloner.Clone(original.algorithm);
+      clonedAlgorithms = cloner.Clone(original.clonedAlgorithms);
+      results = cloner.Clone(original.results);
+
+      folds = cloner.Clone(original.folds);
+      numberOfWorkers = cloner.Clone(original.numberOfWorkers);
+      samplesStart = cloner.Clone(original.samplesStart);
+      samplesEnd = cloner.Clone(original.samplesEnd);
+      shuffleSamples = cloner.Clone(original.shuffleSamples);
+      seed = original.seed;
+
+      RegisterEvents();
+      if (Algorithm != null) RegisterAlgorithmEvents();
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CrossValidation(this, cloner);
+    }
+
+    #endregion
+
+    #region properties
+    [Storable]
+    private IAlgorithm algorithm;
+    public IAlgorithm Algorithm {
+      get { return algorithm; }
+      set {
+        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
+          throw new InvalidOperationException("Changing the algorithm is only allowed if the CrossValidation is stopped or prepared.");
+        if (algorithm != value) {
+          if (value != null && value.Problem != null && !(value.Problem is IDataAnalysisProblem))
+            throw new ArgumentException("Only algorithms with a DataAnalysisProblem could be used for the cross validation.");
+          if (algorithm != null) DeregisterAlgorithmEvents();
+          algorithm = value;
+          Parameters.Clear();
+
+          if (algorithm != null) {
+            algorithm.StoreAlgorithmInEachRun = false;
+            RegisterAlgorithmEvents();
+            algorithm.Prepare(true);
+            Parameters.AddRange(algorithm.Parameters);
+          }
+          OnAlgorithmChanged();
+          Prepare();
+        }
+      }
+    }
+
+
+    [Storable]
+    private IDataAnalysisProblem problem;
+    public IDataAnalysisProblem Problem {
+      get {
+        if (algorithm == null)
+          return null;
+        return (IDataAnalysisProblem)algorithm.Problem;
+      }
+      set {
+        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
+          throw new InvalidOperationException("Changing the problem is only allowed if the CrossValidation is stopped or prepared.");
+        if (algorithm == null) throw new ArgumentNullException("Could not set a problem before an algorithm was set.");
+        algorithm.Problem = value;
+        problem = value;
+      }
+    }
+
+    IProblem IAlgorithm.Problem {
+      get { return Problem; }
+      set {
+        if (value != null && !ProblemType.IsInstanceOfType(value))
+          throw new ArgumentException("Only DataAnalysisProblems could be used for the cross validation.");
+        Problem = (IDataAnalysisProblem)value;
+      }
+    }
+    public Type ProblemType {
+      get { return typeof(IDataAnalysisProblem); }
+    }
+
+    [Storable]
+    private ItemCollection<IAlgorithm> clonedAlgorithms;
+
+    public IEnumerable<IOptimizer> NestedOptimizers {
+      get {
+        if (Algorithm == null) yield break;
+        yield return Algorithm;
+      }
+    }
+
+    [Storable]
+    private ResultCollection results;
+    public ResultCollection Results {
+      get { return results; }
+    }
+    [Storable]
+    private BoolValue shuffleSamples;
+    public BoolValue ShuffleSamples {
+      get { return shuffleSamples; }
+    }
+    [Storable]
+    private IntValue folds;
+    public IntValue Folds {
+      get { return folds; }
+    }
+    [Storable]
+    private IntValue samplesStart;
+    public IntValue SamplesStart {
+      get { return samplesStart; }
+    }
+    [Storable]
+    private IntValue samplesEnd;
+    public IntValue SamplesEnd {
+      get { return samplesEnd; }
+    }
+    [Storable]
+    private IntValue numberOfWorkers;
+    public IntValue NumberOfWorkers {
+      get { return numberOfWorkers; }
+    }
+
+    [Storable]
+    private bool storeAlgorithmInEachRun;
+    public bool StoreAlgorithmInEachRun {
+      get { return storeAlgorithmInEachRun; }
+      set {
+        if (storeAlgorithmInEachRun != value) {
+          storeAlgorithmInEachRun = value;
+          OnStoreAlgorithmInEachRunChanged();
+        }
+      }
+    }
+
+    [Storable]
+    private int runsCounter;
+    [Storable]
+    private RunCollection runs;
+    public RunCollection Runs {
+      get { return runs; }
+    }
+
+    [Storable]
+    private ExecutionState executionState;
+    public ExecutionState ExecutionState {
+      get { return executionState; }
+      private set {
+        if (executionState != value) {
+          executionState = value;
+          OnExecutionStateChanged();
+          OnItemImageChanged();
+        }
+      }
+    }
+    public static new Image StaticItemImage {
+      get { return HeuristicLab.Common.Resources.VSImageLibrary.Event; }
+    }
+    public override Image ItemImage {
+      get {
+        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
+        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
+        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
+        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
+        else return base.ItemImage;
+      }
+    }
+
+    public TimeSpan ExecutionTime {
+      get {
+        if (ExecutionState != ExecutionState.Prepared)
+          return TimeSpan.FromMilliseconds(clonedAlgorithms.Select(x => x.ExecutionTime.TotalMilliseconds).Sum());
+        return TimeSpan.Zero;
+      }
+    }
+    #endregion
+
+    protected override void OnNameChanged() {
+      base.OnNameChanged();
+      Runs.OptimizerName = Name;
+    }
+
+    public void Prepare() {
+      if (startPending) return;
+      if (ExecutionState == ExecutionState.Started)
+        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
+      results.Clear();
+      clonedAlgorithms.Clear();
+      if (Algorithm != null) {
+        Algorithm.Prepare();
+        if (Algorithm.ExecutionState == ExecutionState.Prepared) OnPrepared();
+      }
+    }
+    public void Prepare(bool clearRuns) {
+      if (clearRuns) runs.Clear();
+      Prepare();
+    }
+
+    private bool startPending;
+    public void Start() {
+      Start(CancellationToken.None);
+    }
+    public void Start(CancellationToken cancellationToken) {
+      lock (locker) {
+        if (startPending) return;
+        startPending = true;
+      }
+
+      try {
+        if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
+          throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));
+        seed = RandomSeedGenerator.GetSeed();
+
+        if (Algorithm == null) return;
+        //create cloned algorithms
+        if (clonedAlgorithms.Count == 0) {
+          int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / Folds.Value;
+          IDataset shuffledDataset = null;
+          for (int i = 0; i < Folds.Value; i++) {
+            var cloner = new Cloner();
+            if (ShuffleSamples.Value) {
+              var random = new FastRandom(seed);
+              var dataAnalysisProblem = (IDataAnalysisProblem)algorithm.Problem;
+              var dataset = (Dataset)dataAnalysisProblem.ProblemData.Dataset;
+              shuffledDataset = shuffledDataset ?? dataset.Shuffle(random);
+              cloner.RegisterClonedObject(dataset, shuffledDataset);
+            }
+            IAlgorithm clonedAlgorithm = cloner.Clone(Algorithm);
+            clonedAlgorithm.Name = algorithm.Name + " Fold " + i;
+            IDataAnalysisProblem problem = clonedAlgorithm.Problem as IDataAnalysisProblem;
+            ISymbolicDataAnalysisProblem symbolicProblem = problem as ISymbolicDataAnalysisProblem;
+
+            int testStart = (i * testSamplesCount) + SamplesStart.Value;
+            int testEnd = (i + 1) == Folds.Value ? SamplesEnd.Value : (i + 1) * testSamplesCount + SamplesStart.Value;
+
+            problem.ProblemData.TrainingPartition.Start = SamplesStart.Value;
+            problem.ProblemData.TrainingPartition.End = SamplesEnd.Value;
+            problem.ProblemData.TestPartition.Start = testStart;
+            problem.ProblemData.TestPartition.End = testEnd;
+            DataAnalysisProblemData problemData = problem.ProblemData as DataAnalysisProblemData;
+            if (problemData != null) {
+              problemData.TrainingPartitionParameter.Hidden = false;
+              problemData.TestPartitionParameter.Hidden = false;
+            }
+
+            if (symbolicProblem != null) {
+              symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
+              symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
+            }
+            clonedAlgorithm.Prepare();
+            clonedAlgorithms.Add(clonedAlgorithm);
+          }
+        }
+
+        OnStarted();
+      } finally {
+        if (startPending) startPending = false;
+      }
+
+      availableWorkers = new SemaphoreSlim(NumberOfWorkers.Value, NumberOfWorkers.Value);
+      allAlgorithmsFinished = new ManualResetEventSlim(false);
+
+      var startedTasks = new List<Task>(clonedAlgorithms.Count);
+
+      //start prepared or paused cloned algorithms
+      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
+        if (pausePending || stopPending || ExecutionState != ExecutionState.Started) break;
+        if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared ||
+            clonedAlgorithm.ExecutionState == ExecutionState.Paused) {
+          availableWorkers.Wait();
+          lock (locker) {
+            if (pausePending || stopPending || ExecutionState != ExecutionState.Started) break;
+            var task = clonedAlgorithm.StartAsync(cancellationToken);
+            startedTasks.Add(task);
+          }
+        }
+      }
+
+      allAlgorithmsFinished.Wait();
+
+      Task.WaitAll(startedTasks.ToArray()); // to get exceptions not handled within the tasks
+    }
+
+    public async Task StartAsync() { await StartAsync(CancellationToken.None); }
+    public async Task StartAsync(CancellationToken cancellationToken) {
+      await AsyncHelper.DoAsync(Start, cancellationToken);
+    }
+
+    private bool pausePending;
+    public void Pause() {
+      if (startPending) return;
+      if (ExecutionState != ExecutionState.Started)
+        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
+      if (!pausePending) {
+        pausePending = true;
+        lock (locker) {
+          var toPause = clonedAlgorithms.Where(x => x.ExecutionState == ExecutionState.Started).ToList();
+          foreach (var optimizer in toPause) {
+            // a race-condition may occur when the optimizer has changed the state by itself in the meantime
+            try { optimizer.Pause(); } catch (InvalidOperationException) { }
+          }
+        }
+      }
+    }
+
+    private bool stopPending;
+    public void Stop() {
+      if (startPending) return;
+      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
+        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".",
+                                                          ExecutionState));
+      if (!stopPending) {
+        stopPending = true;
+        lock (locker) {
+          var toStop = clonedAlgorithms.Where(x => x.ExecutionState == ExecutionState.Started || x.ExecutionState == ExecutionState.Paused).ToList();
+          foreach (var optimizer in toStop) {
+            // a race-condition may occur when the optimizer has changed the state by itself in the meantime
+            try { optimizer.Stop(); } catch (InvalidOperationException) { }
+          }
+        }
+      }
+    }
+
+    #region collect parameters and results
+    public override void CollectParameterValues(IDictionary<string, IItem> values) {
+      values.Add("Algorithm Name", new StringValue(Name));
+      values.Add("Algorithm Type", new StringValue(GetType().GetPrettyName()));
+      values.Add("Folds", new IntValue(Folds.Value));
+
+      if (algorithm != null) {
+        values.Add("CrossValidation Algorithm Name", new StringValue(Algorithm.Name));
+        values.Add("CrossValidation Algorithm Type", new StringValue(Algorithm.GetType().GetPrettyName()));
+        base.CollectParameterValues(values);
+      }
+      if (Problem != null) {
+        values.Add("Problem Name", new StringValue(Problem.Name));
+        values.Add("Problem Type", new StringValue(Problem.GetType().GetPrettyName()));
+        Problem.CollectParameterValues(values);
+      }
+    }
+
+    public void CollectResultValues(IDictionary<string, IItem> results) {
+      var clonedResults = (ResultCollection)this.results.Clone();
+      foreach (var result in clonedResults) {
+        results.Add(result.Name, result.Value);
+      }
+    }
+
+    private void AggregateResultValues(IDictionary<string, IItem> results) {
+      IEnumerable<IRun> runs = clonedAlgorithms.Select(alg => alg.Runs.FirstOrDefault()).Where(run => run != null);
+      IEnumerable<KeyValuePair<string, IItem>> resultCollections = runs.Where(x => x != null).SelectMany(x => x.Results).ToList();
+
+      foreach (IResult result in ExtractAndAggregateResults<IntValue>(resultCollections))
+        results.Add(result.Name, result.Value);
+      foreach (IResult result in ExtractAndAggregateResults<DoubleValue>(resultCollections))
+        results.Add(result.Name, result.Value);
+      foreach (IResult result in ExtractAndAggregateResults<PercentValue>(resultCollections))
+        results.Add(result.Name, result.Value);
+      foreach (IResult result in ExtractAndAggregateRegressionSolutions(resultCollections)) {
+        results.Add(result.Name, result.Value);
+      }
+      foreach (IResult result in ExtractAndAggregateClassificationSolutions(resultCollections)) {
+        results.Add(result.Name, result.Value);
+      }
+      results.Add("Execution Time", new TimeSpanValue(this.ExecutionTime));
+      results.Add("CrossValidation Folds", new RunCollection(runs));
+    }
+
+    private IEnumerable<IResult> ExtractAndAggregateRegressionSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
+      Dictionary<string, List<IRegressionSolution>> resultSolutions = new Dictionary<string, List<IRegressionSolution>>();
+      foreach (var result in resultCollections) {
+        var regressionSolution = result.Value as IRegressionSolution;
+        if (regressionSolution != null) {
+          if (resultSolutions.ContainsKey(result.Key)) {
+            resultSolutions[result.Key].Add(regressionSolution);
+          } else {
+            resultSolutions.Add(result.Key, new List<IRegressionSolution>() { regressionSolution });
+          }
+        }
+      }
+      List<IResult> aggregatedResults = new List<IResult>();
+      foreach (KeyValuePair<string, List<IRegressionSolution>> solutions in resultSolutions) {
+        // clone manually to correctly clone references between cloned root objects 
+        Cloner cloner = new Cloner();
+        if (ShuffleSamples.Value) {
+          var dataset = (Dataset)Problem.ProblemData.Dataset;
+          var random = new FastRandom(seed);
+          var shuffledDataset = dataset.Shuffle(random);
+          cloner.RegisterClonedObject(dataset, shuffledDataset);
+        }
+        var problemDataClone = (IRegressionProblemData)cloner.Clone(Problem.ProblemData);
+        // set partitions of problem data clone correctly
+        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
+        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
+        // clone models
+        var ensembleSolution = new RegressionEnsembleSolution(problemDataClone);
+        ensembleSolution.AddRegressionSolutions(solutions.Value);
+
+        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
+      }
+      List<IResult> flattenedResults = new List<IResult>();
+      CollectResultsRecursively("", aggregatedResults, flattenedResults);
+      return flattenedResults;
+    }
+
+    private IEnumerable<IResult> ExtractAndAggregateClassificationSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
+      Dictionary<string, List<IClassificationSolution>> resultSolutions = new Dictionary<string, List<IClassificationSolution>>();
+      foreach (var result in resultCollections) {
+        var classificationSolution = result.Value as IClassificationSolution;
+        if (classificationSolution != null) {
+          if (resultSolutions.ContainsKey(result.Key)) {
+            resultSolutions[result.Key].Add(classificationSolution);
+          } else {
+            resultSolutions.Add(result.Key, new List<IClassificationSolution>() { classificationSolution });
+          }
+        }
+      }
+      var aggregatedResults = new List<IResult>();
+      foreach (KeyValuePair<string, List<IClassificationSolution>> solutions in resultSolutions) {
+        // at least one algorithm (GBT with logistic regression loss) produces a classification solution even though the original problem is a regression problem.
+        var targetVariable = solutions.Value.First().ProblemData.TargetVariable;
+        var dataset = (Dataset)Problem.ProblemData.Dataset;
+        if (ShuffleSamples.Value) {
+          var random = new FastRandom(seed);
+          dataset = dataset.Shuffle(random);
+        }
+        var problemDataClone = new ClassificationProblemData(dataset, Problem.ProblemData.AllowedInputVariables, targetVariable);
+        // set partitions of problem data clone correctly
+        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
+        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
+        // clone models
+        var ensembleSolution = new ClassificationEnsembleSolution(problemDataClone);
+        ensembleSolution.AddClassificationSolutions(solutions.Value);
+
+        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
+      }
+      List<IResult> flattenedResults = new List<IResult>();
+      CollectResultsRecursively("", aggregatedResults, flattenedResults);
+      return flattenedResults;
+    }
+
+    private void CollectResultsRecursively(string path, IEnumerable<IResult> results, IList<IResult> flattenedResults) {
+      foreach (IResult result in results) {
+        flattenedResults.Add(new Result(path + result.Name, result.Value));
+        ResultCollection childCollection = result.Value as ResultCollection;
+        if (childCollection != null) {
+          CollectResultsRecursively(path + result.Name + ".", childCollection, flattenedResults);
+        }
+      }
+    }
+
+    private static IEnumerable<IResult> ExtractAndAggregateResults<T>(IEnumerable<KeyValuePair<string, IItem>> results)
+  where T : class, IItem, new() {
+      Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
+      foreach (var resultValue in results.Where(r => r.Value.GetType() == typeof(T))) {
+        if (!resultValues.ContainsKey(resultValue.Key))
+          resultValues[resultValue.Key] = new List<double>();
+        resultValues[resultValue.Key].Add(ConvertToDouble(resultValue.Value));
+      }
+
+      DoubleValue doubleValue;
+      if (typeof(T) == typeof(PercentValue))
+        doubleValue = new PercentValue();
+      else if (typeof(T) == typeof(DoubleValue))
+        doubleValue = new DoubleValue();
+      else if (typeof(T) == typeof(IntValue))
+        doubleValue = new DoubleValue();
+      else
+        throw new NotSupportedException();
+
+      List<IResult> aggregatedResults = new List<IResult>();
+      foreach (KeyValuePair<string, List<double>> resultValue in resultValues) {
+        doubleValue.Value = resultValue.Value.Average();
+        aggregatedResults.Add(new Result(resultValue.Key + " (average)", (IItem)doubleValue.Clone()));
+        doubleValue.Value = resultValue.Value.StandardDeviation();
+        aggregatedResults.Add(new Result(resultValue.Key + " (std.dev.)", (IItem)doubleValue.Clone()));
+      }
+      return aggregatedResults;
+    }
+
+    private static double ConvertToDouble(IItem item) {
+      if (item is DoubleValue) return ((DoubleValue)item).Value;
+      else if (item is IntValue) return ((IntValue)item).Value;
+      else throw new NotSupportedException("Could not convert any item type to double");
+    }
+    #endregion
+
+    #region events
+    private void RegisterEvents() {
+      Folds.ValueChanged += new EventHandler(Folds_ValueChanged);
+      RegisterClonedAlgorithmsEvents();
+    }
+    private void Folds_ValueChanged(object sender, EventArgs e) {
+      if (ExecutionState != ExecutionState.Prepared)
+        throw new InvalidOperationException("Can not change number of folds if the execution state is not prepared.");
+    }
+
+
+    #region template algorithms events
+    public event EventHandler AlgorithmChanged;
+    private void OnAlgorithmChanged() {
+      EventHandler handler = AlgorithmChanged;
+      if (handler != null) handler(this, EventArgs.Empty);
+      OnProblemChanged();
+      if (Problem == null) ExecutionState = ExecutionState.Stopped;
+    }
+    private void RegisterAlgorithmEvents() {
+      algorithm.ProblemChanged += new EventHandler(Algorithm_ProblemChanged);
+      algorithm.ExecutionStateChanged += new EventHandler(Algorithm_ExecutionStateChanged);
+      if (Problem != null) {
+        Problem.Reset += new EventHandler(Problem_Reset);
+      }
+    }
+    private void DeregisterAlgorithmEvents() {
+      algorithm.ProblemChanged -= new EventHandler(Algorithm_ProblemChanged);
+      algorithm.ExecutionStateChanged -= new EventHandler(Algorithm_ExecutionStateChanged);
+      if (Problem != null) {
+        Problem.Reset -= new EventHandler(Problem_Reset);
+      }
+    }
+    private void Algorithm_ProblemChanged(object sender, EventArgs e) {
+      if (algorithm.Problem != null && !(algorithm.Problem is IDataAnalysisProblem)) {
+        algorithm.Problem = problem;
+        throw new ArgumentException("A cross validation algorithm can only contain DataAnalysisProblems.");
+      }
+      if (problem != null) problem.Reset -= new EventHandler(Problem_Reset);
+      problem = (IDataAnalysisProblem)algorithm.Problem;
+      if (problem != null) problem.Reset += new EventHandler(Problem_Reset);
+      OnProblemChanged();
+    }
+    public event EventHandler ProblemChanged;
+    private void OnProblemChanged() {
+      EventHandler handler = ProblemChanged;
+      if (handler != null) handler(this, EventArgs.Empty);
+      ConfigureProblem();
+    }
+    private void Problem_Reset(object sender, EventArgs e) {
+      ConfigureProblem();
+    }
+    private void ConfigureProblem() {
+      SamplesStart.Value = 0;
+      if (Problem != null) {
+        SamplesEnd.Value = Problem.ProblemData.Dataset.Rows;
+
+        DataAnalysisProblemData problemData = Problem.ProblemData as DataAnalysisProblemData;
+        if (problemData != null) {
+          problemData.TrainingPartitionParameter.Hidden = true;
+          problemData.TestPartitionParameter.Hidden = true;
+        }
+        ISymbolicDataAnalysisProblem symbolicProblem = Problem as ISymbolicDataAnalysisProblem;
+        if (symbolicProblem != null) {
+          symbolicProblem.FitnessCalculationPartitionParameter.Hidden = true;
+          symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
+          symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
+          symbolicProblem.ValidationPartitionParameter.Hidden = true;
+          symbolicProblem.ValidationPartition.Start = 0;
+          symbolicProblem.ValidationPartition.End = 0;
+        }
+      } else
+        SamplesEnd.Value = 0;
+    }
+
+    private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
+      switch (Algorithm.ExecutionState) {
+        case ExecutionState.Prepared:
+          OnPrepared();
+          break;
+        case ExecutionState.Started: throw new InvalidOperationException("Algorithm template can not be started.");
+        case ExecutionState.Paused: throw new InvalidOperationException("Algorithm template can not be paused.");
+        case ExecutionState.Stopped:
+          OnStopped();
+          break;
+      }
+    }
+    #endregion
+
+    #region clonedAlgorithms events
+    private void RegisterClonedAlgorithmsEvents() {
+      clonedAlgorithms.ItemsAdded += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
+      clonedAlgorithms.ItemsRemoved += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
+      clonedAlgorithms.CollectionReset += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
+      foreach (IAlgorithm algorithm in clonedAlgorithms)
+        RegisterClonedAlgorithmEvents(algorithm);
+    }
+    private void DeregisterClonedAlgorithmsEvents() {
+      clonedAlgorithms.ItemsAdded -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
+      clonedAlgorithms.ItemsRemoved -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
+      clonedAlgorithms.CollectionReset -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
+      foreach (IAlgorithm algorithm in clonedAlgorithms)
+        DeregisterClonedAlgorithmEvents(algorithm);
+    }
+    private void ClonedAlgorithms_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
+      foreach (IAlgorithm algorithm in e.Items)
+        RegisterClonedAlgorithmEvents(algorithm);
+    }
+    private void ClonedAlgorithms_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
+      foreach (IAlgorithm algorithm in e.Items)
+        DeregisterClonedAlgorithmEvents(algorithm);
+    }
+    private void ClonedAlgorithms_CollectionReset(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
+      foreach (IAlgorithm algorithm in e.OldItems)
+        DeregisterClonedAlgorithmEvents(algorithm);
+      foreach (IAlgorithm algorithm in e.Items)
+        RegisterClonedAlgorithmEvents(algorithm);
+    }
+    private void RegisterClonedAlgorithmEvents(IAlgorithm algorithm) {
+      algorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
+      algorithm.ExecutionTimeChanged += new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
+      algorithm.Started += new EventHandler(ClonedAlgorithm_Started);
+      algorithm.Paused += new EventHandler(ClonedAlgorithm_Paused);
+      algorithm.Stopped += new EventHandler(ClonedAlgorithm_Stopped);
+    }
+    private void DeregisterClonedAlgorithmEvents(IAlgorithm algorithm) {
+      algorithm.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
+      algorithm.ExecutionTimeChanged -= new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
+      algorithm.Started -= new EventHandler(ClonedAlgorithm_Started);
+      algorithm.Paused -= new EventHandler(ClonedAlgorithm_Paused);
+      algorithm.Stopped -= new EventHandler(ClonedAlgorithm_Stopped);
+    }
+    private void ClonedAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
+      Pause();
+      OnExceptionOccurred(e.Value);
+    }
+    private void ClonedAlgorithm_ExecutionTimeChanged(object sender, EventArgs e) {
+      OnExecutionTimeChanged();
+    }
+
+    private readonly object locker = new object();
+    private readonly object resultLocker = new object();
+    private void ClonedAlgorithm_Started(object sender, EventArgs e) {
+      IAlgorithm algorithm = sender as IAlgorithm;
+      lock (resultLocker) {
+        if (algorithm != null && !results.ContainsKey(algorithm.Name))
+          results.Add(new Result(algorithm.Name, "Contains results for the specific fold.", algorithm.Results));
+      }
+    }
+
+    private void ClonedAlgorithm_Paused(object sender, EventArgs e) {
+      lock (locker) {
+        availableWorkers.Release();
+        if (clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started)) {
+          OnPaused();
+          allAlgorithmsFinished.Set();
+        }
+      }
+    }
+
+    private void ClonedAlgorithm_Stopped(object sender, EventArgs e) {
+      lock (locker) {
+        // if the algorithm was in paused state, its worker has already been released
+        if (availableWorkers.CurrentCount < NumberOfWorkers.Value)
+          availableWorkers.Release();
+        if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped)) {
+          OnStopped();
+          allAlgorithmsFinished.Set();
+        } else if (stopPending && clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped)) {
+          OnStopped();
+          allAlgorithmsFinished.Set();
+        }
+      }
+    }
+    #endregion
+    #endregion
+
+    #region event firing
+    public event EventHandler ExecutionStateChanged;
+    private void OnExecutionStateChanged() {
+      EventHandler handler = ExecutionStateChanged;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
+    public event EventHandler ExecutionTimeChanged;
+    private void OnExecutionTimeChanged() {
+      EventHandler handler = ExecutionTimeChanged;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
+    public event EventHandler Prepared;
+    private void OnPrepared() {
+      ExecutionState = ExecutionState.Prepared;
+      EventHandler handler = Prepared;
+      if (handler != null) handler(this, EventArgs.Empty);
+      OnExecutionTimeChanged();
+    }
+    public event EventHandler Started;
+    private void OnStarted() {
+      startPending = false;
+      ExecutionState = ExecutionState.Started;
+      EventHandler handler = Started;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
+    public event EventHandler Paused;
+    private void OnPaused() {
+      pausePending = false;
+      ExecutionState = ExecutionState.Paused;
+      EventHandler handler = Paused;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
+    public event EventHandler Stopped;
+    private void OnStopped() {
+      stopPending = false;
+      Dictionary<string, IItem> collectedResults = new Dictionary<string, IItem>();
+      AggregateResultValues(collectedResults);
+      results.AddRange(collectedResults.Select(x => new Result(x.Key, x.Value)).Cast<IResult>().ToArray());
+      clonedAlgorithms.Clear();
+      runsCounter++;
+      runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
+      ExecutionState = ExecutionState.Stopped;
+      EventHandler handler = Stopped;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
+    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
+    private void OnExceptionOccurred(Exception exception) {
+      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
+      if (handler != null) handler(this, new EventArgs<Exception>(exception));
+    }
+    public event EventHandler StoreAlgorithmInEachRunChanged;
+    private void OnStoreAlgorithmInEachRunChanged() {
+      EventHandler handler = StoreAlgorithmInEachRunChanged;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/DoubleArrayExtensions.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/DoubleArrayExtensions.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/DoubleArrayExtensions.cs	(revision 17041)
@@ -0,0 +1,12 @@
+﻿namespace HeuristicLab.Algorithms.DataAnalysis {
+  public static class DoubleArrayExtensions {
+    public static bool ContainsNanOrInfinity(this double[,] array) {
+      foreach (var entry in array) {
+        if (double.IsNaN(entry) || double.IsInfinity(entry)) {
+          return true;
+        }
+      }
+      return false;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/FixedDataAnalysisAlgorithm.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/FixedDataAnalysisAlgorithm.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/FixedDataAnalysisAlgorithm.cs	(revision 17041)
@@ -0,0 +1,49 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using HeuristicLab.Common;
+using HeuristicLab.Optimization;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("E7BEE142-B6F1-4293-AC2A-B19EB163AB2E")]
+  public abstract class FixedDataAnalysisAlgorithm<T> : BasicAlgorithm, IDataAnalysisAlgorithm<T> where T : class, IDataAnalysisProblem {
+    #region Properties
+    public override Type ProblemType {
+      get { return typeof(T); }
+    }
+    public new T Problem {
+      get { return (T)base.Problem; }
+      set { base.Problem = value; }
+    }
+    #endregion
+
+    public override bool SupportsPause { get { return false; } }
+
+    [StorableConstructor]
+    protected FixedDataAnalysisAlgorithm(StorableConstructorFlag _) : base(_) { }
+    protected FixedDataAnalysisAlgorithm(FixedDataAnalysisAlgorithm<T> original, Cloner cloner) : base(original, cloner) { }
+    public FixedDataAnalysisAlgorithm() : base() { }
+
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GBM/GradientBoostingRegressionAlgorithm.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GBM/GradientBoostingRegressionAlgorithm.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GBM/GradientBoostingRegressionAlgorithm.cs	(revision 17041)
@@ -0,0 +1,506 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Analysis;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
+using HeuristicLab.Random;
+using HeuristicLab.Selection;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression {
+  [Item("Gradient Boosting Machine Regression (GBM)",
+    "Gradient boosting for any regression base learner (e.g. MCTS symbolic regression)")]
+  [StorableType("98B340D7-DB23-40F9-A9CC-C3E652E92671")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 350)]
+  public class GradientBoostingRegressionAlgorithm : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+
+    #region ParameterNames
+
+    private const string IterationsParameterName = "Iterations";
+    private const string NuParameterName = "Nu";
+    private const string MParameterName = "M";
+    private const string RParameterName = "R";
+    private const string RegressionAlgorithmParameterName = "RegressionAlgorithm";
+    private const string SeedParameterName = "Seed";
+    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+    private const string CreateSolutionParameterName = "CreateSolution";
+    private const string StoreRunsParameterName = "StoreRuns";
+    private const string RegressionAlgorithmSolutionResultParameterName = "RegressionAlgorithmResult";
+
+    #endregion
+
+    #region ParameterProperties
+
+    public IFixedValueParameter<IntValue> IterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[IterationsParameterName]; }
+    }
+
+    public IFixedValueParameter<DoubleValue> NuParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[NuParameterName]; }
+    }
+
+    public IFixedValueParameter<DoubleValue> RParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[RParameterName]; }
+    }
+
+    public IFixedValueParameter<DoubleValue> MParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[MParameterName]; }
+    }
+
+    // regression algorithms are currently: DataAnalysisAlgorithms, BasicAlgorithms and engine algorithms with no common interface
+    public IConstrainedValueParameter<IAlgorithm> RegressionAlgorithmParameter {
+      get { return (IConstrainedValueParameter<IAlgorithm>)Parameters[RegressionAlgorithmParameterName]; }
+    }
+
+    public IFixedValueParameter<StringValue> RegressionAlgorithmSolutionResultParameter {
+      get { return (IFixedValueParameter<StringValue>)Parameters[RegressionAlgorithmSolutionResultParameterName]; }
+    }
+
+    public IFixedValueParameter<IntValue> SeedParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+
+    public FixedValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (FixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+
+    public IFixedValueParameter<BoolValue> CreateSolutionParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> StoreRunsParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[StoreRunsParameterName]; }
+    }
+
+    #endregion
+
+    #region Properties
+
+    public int Iterations {
+      get { return IterationsParameter.Value.Value; }
+      set { IterationsParameter.Value.Value = value; }
+    }
+
+    public int Seed {
+      get { return SeedParameter.Value.Value; }
+      set { SeedParameter.Value.Value = value; }
+    }
+
+    public bool SetSeedRandomly {
+      get { return SetSeedRandomlyParameter.Value.Value; }
+      set { SetSeedRandomlyParameter.Value.Value = value; }
+    }
+
+    public double Nu {
+      get { return NuParameter.Value.Value; }
+      set { NuParameter.Value.Value = value; }
+    }
+
+    public double R {
+      get { return RParameter.Value.Value; }
+      set { RParameter.Value.Value = value; }
+    }
+
+    public double M {
+      get { return MParameter.Value.Value; }
+      set { MParameter.Value.Value = value; }
+    }
+
+    public bool CreateSolution {
+      get { return CreateSolutionParameter.Value.Value; }
+      set { CreateSolutionParameter.Value.Value = value; }
+    }
+
+    public bool StoreRuns {
+      get { return StoreRunsParameter.Value.Value; }
+      set { StoreRunsParameter.Value.Value = value; }
+    }
+
+    public IAlgorithm RegressionAlgorithm {
+      get { return RegressionAlgorithmParameter.Value; }
+    }
+
+    public string RegressionAlgorithmResult {
+      get { return RegressionAlgorithmSolutionResultParameter.Value.Value; }
+      set { RegressionAlgorithmSolutionResultParameter.Value.Value = value; }
+    }
+
+    #endregion
+
+    [StorableConstructor]
+    protected GradientBoostingRegressionAlgorithm(StorableConstructorFlag _) : base(_) {
+    }
+
+    protected GradientBoostingRegressionAlgorithm(GradientBoostingRegressionAlgorithm original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GradientBoostingRegressionAlgorithm(this, cloner);
+    }
+
+    public GradientBoostingRegressionAlgorithm() {
+      Problem = new RegressionProblem(); // default problem
+      var osgp = CreateOSGP();
+      var regressionAlgs = new ItemSet<IAlgorithm>(new IAlgorithm[] {
+        new RandomForestRegression(),
+        osgp,
+      });
+      foreach (var alg in regressionAlgs) alg.Prepare();
+
+
+      Parameters.Add(new FixedValueParameter<IntValue>(IterationsParameterName,
+        "Number of iterations", new IntValue(100)));
+      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName,
+        "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName,
+        "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(NuParameterName,
+        "The learning rate nu when updating predictions in GBM (0 < nu <= 1)", new DoubleValue(0.5)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(RParameterName,
+        "The fraction of rows that are sampled randomly for the base learner in each iteration (0 < r <= 1)",
+        new DoubleValue(1)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName,
+        "The fraction of variables that are sampled randomly for the base learner in each iteration (0 < m <= 1)",
+        new DoubleValue(0.5)));
+      Parameters.Add(new ConstrainedValueParameter<IAlgorithm>(RegressionAlgorithmParameterName,
+        "The regression algorithm to use as a base learner", regressionAlgs, osgp));
+      Parameters.Add(new FixedValueParameter<StringValue>(RegressionAlgorithmSolutionResultParameterName,
+        "The name of the solution produced by the regression algorithm", new StringValue("Solution")));
+      Parameters[RegressionAlgorithmSolutionResultParameterName].Hidden = true;
+      Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName,
+        "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+      Parameters[CreateSolutionParameterName].Hidden = true;
+      Parameters.Add(new FixedValueParameter<BoolValue>(StoreRunsParameterName,
+        "Flag that indicates if the results of the individual runs should be stored for detailed analysis", new BoolValue(false)));
+      Parameters[StoreRunsParameterName].Hidden = true;
+    }
+
+    protected override void Run(CancellationToken cancellationToken) {
+      // Set up the algorithm
+      if (SetSeedRandomly) Seed = RandomSeedGenerator.GetSeed();
+      var rand = new MersenneTwister((uint)Seed);
+
+      // Set up the results display
+      var iterations = new IntValue(0);
+      Results.Add(new Result("Iterations", iterations));
+
+      var table = new DataTable("Qualities");
+      table.Rows.Add(new DataRow("R² (train)"));
+      table.Rows.Add(new DataRow("R² (test)"));
+      Results.Add(new Result("Qualities", table));
+      var curLoss = new DoubleValue();
+      var curTestLoss = new DoubleValue();
+      Results.Add(new Result("R² (train)", curLoss));
+      Results.Add(new Result("R² (test)", curTestLoss));
+      var runCollection = new RunCollection();
+      if (StoreRuns)
+        Results.Add(new Result("Runs", runCollection));
+
+      // init
+      var problemData = Problem.ProblemData;
+      var targetVarName = problemData.TargetVariable;
+      var activeVariables = problemData.AllowedInputVariables.Concat(new string[] { problemData.TargetVariable });
+      var modifiableDataset = new ModifiableDataset(
+        activeVariables,
+        activeVariables.Select(v => problemData.Dataset.GetDoubleValues(v).ToList()));
+
+      var trainingRows = problemData.TrainingIndices;
+      var testRows = problemData.TestIndices;
+      var yPred = new double[trainingRows.Count()];
+      var yPredTest = new double[testRows.Count()];
+      var y = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices).ToArray();
+      var curY = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices).ToArray();
+
+      var yTest = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TestIndices).ToArray();
+      var curYTest = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TestIndices).ToArray();
+      var nu = Nu;
+      var mVars = (int)Math.Ceiling(M * problemData.AllowedInputVariables.Count());
+      var rRows = (int)Math.Ceiling(R * problemData.TrainingIndices.Count());
+      var alg = RegressionAlgorithm;
+      List<IRegressionModel> models = new List<IRegressionModel>();
+      try {
+
+        // Loop until iteration limit reached or canceled.
+        for (int i = 0; i < Iterations; i++) {
+          cancellationToken.ThrowIfCancellationRequested();
+
+          modifiableDataset.RemoveVariable(targetVarName);
+          modifiableDataset.AddVariable(targetVarName, curY.Concat(curYTest).ToList());
+
+          SampleTrainingData(rand, modifiableDataset, rRows, problemData.Dataset, curY, problemData.TargetVariable, problemData.TrainingIndices); // all training indices from the original problem data are allowed 
+          var modifiableProblemData = new RegressionProblemData(modifiableDataset,
+            problemData.AllowedInputVariables.SampleRandomWithoutRepetition(rand, mVars),
+            problemData.TargetVariable);
+          modifiableProblemData.TrainingPartition.Start = 0;
+          modifiableProblemData.TrainingPartition.End = rRows;
+          modifiableProblemData.TestPartition.Start = problemData.TestPartition.Start;
+          modifiableProblemData.TestPartition.End = problemData.TestPartition.End;
+
+          if (!TrySetProblemData(alg, modifiableProblemData))
+            throw new NotSupportedException("The algorithm cannot be used with GBM.");
+
+          IRegressionModel model;
+          IRun run;
+
+          // try to find a model. The algorithm might fail to produce a model. In this case we just retry until the iterations are exhausted
+          if (TryExecute(alg, rand.Next(), RegressionAlgorithmResult, out model, out run)) {
+            int row = 0;
+            // update predictions for training and test
+            // update new targets (in the case of squared error loss we simply use negative residuals)
+            foreach (var pred in model.GetEstimatedValues(problemData.Dataset, trainingRows)) {
+              yPred[row] = yPred[row] + nu * pred;
+              curY[row] = y[row] - yPred[row];
+              row++;
+            }
+            row = 0;
+            foreach (var pred in model.GetEstimatedValues(problemData.Dataset, testRows)) {
+              yPredTest[row] = yPredTest[row] + nu * pred;
+              curYTest[row] = yTest[row] - yPredTest[row];
+              row++;
+            }
+            // determine quality
+            OnlineCalculatorError error;
+            var trainR = OnlinePearsonsRCalculator.Calculate(yPred, y, out error);
+            var testR = OnlinePearsonsRCalculator.Calculate(yPredTest, yTest, out error);
+
+            // iteration results
+            curLoss.Value = error == OnlineCalculatorError.None ? trainR * trainR : 0.0;
+            curTestLoss.Value = error == OnlineCalculatorError.None ? testR * testR : 0.0;
+
+            models.Add(model);
+
+
+          }
+
+          if (StoreRuns)
+            runCollection.Add(run);
+          table.Rows["R² (train)"].Values.Add(curLoss.Value);
+          table.Rows["R² (test)"].Values.Add(curTestLoss.Value);
+          iterations.Value = i + 1;
+        }
+
+        // produce solution 
+        if (CreateSolution) {
+          // when all our models are symbolic models we can easily combine them to a single model
+          if (models.All(m => m is ISymbolicRegressionModel)) {
+            Results.Add(new Result("Solution", CreateSymbolicSolution(models, Nu, (IRegressionProblemData)problemData.Clone())));
+          }
+          // just produce an ensemble solution for now (TODO: correct scaling or linear regression for ensemble model weights)
+
+          var ensembleSolution = CreateEnsembleSolution(models, (IRegressionProblemData)problemData.Clone());
+          Results.Add(new Result("EnsembleSolution", ensembleSolution));
+        }
+      }
+      finally {
+        // reset everything
+        alg.Prepare(true);
+      }
+    }
+
+    private static IRegressionEnsembleSolution CreateEnsembleSolution(List<IRegressionModel> models,
+      IRegressionProblemData problemData) {
+      var rows = problemData.TrainingPartition.Size;
+      var features = models.Count;
+      double[,] inputMatrix = new double[rows, features + 1];
+      //add model estimates
+      for (int m = 0; m < models.Count; m++) {
+        var model = models[m];
+        var estimates = model.GetEstimatedValues(problemData.Dataset, problemData.TrainingIndices);
+        int estimatesCounter = 0;
+        foreach (var estimate in estimates) {
+          inputMatrix[estimatesCounter, m] = estimate;
+          estimatesCounter++;
+        }
+      }
+
+      //add target
+      var targets = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+      int targetCounter = 0;
+      foreach (var target in targets) {
+        inputMatrix[targetCounter, models.Count] = target;
+        targetCounter++;
+      }
+
+      alglib.linearmodel lm = new alglib.linearmodel();
+      alglib.lrreport ar = new alglib.lrreport();
+      double[] coefficients;
+      int retVal = 1;
+      alglib.lrbuildz(inputMatrix, rows, features, out retVal, out lm, out ar);
+      if (retVal != 1) throw new ArgumentException("Error in calculation of linear regression solution");
+
+      alglib.lrunpack(lm, out coefficients, out features);
+
+      var ensembleModel = new RegressionEnsembleModel(models, coefficients.Take(models.Count)) { AverageModelEstimates = false };
+      var ensembleSolution = (IRegressionEnsembleSolution)ensembleModel.CreateRegressionSolution(problemData);
+      return ensembleSolution;
+    }
+
+
+    private IAlgorithm CreateOSGP() {
+      // configure strict osgp
+      var alg = new OffspringSelectionGeneticAlgorithm.OffspringSelectionGeneticAlgorithm();
+      var prob = new SymbolicRegressionSingleObjectiveProblem();
+      prob.MaximumSymbolicExpressionTreeDepth.Value = 7;
+      prob.MaximumSymbolicExpressionTreeLength.Value = 15;
+      alg.Problem = prob;
+      alg.SuccessRatio.Value = 1.0;
+      alg.ComparisonFactorLowerBound.Value = 1.0;
+      alg.ComparisonFactorUpperBound.Value = 1.0;
+      alg.MutationProbability.Value = 0.15;
+      alg.PopulationSize.Value = 200;
+      alg.MaximumSelectionPressure.Value = 100;
+      alg.MaximumEvaluatedSolutions.Value = 20000;
+      alg.SelectorParameter.Value = alg.SelectorParameter.ValidValues.OfType<GenderSpecificSelector>().First();
+      alg.MutatorParameter.Value = alg.MutatorParameter.ValidValues.OfType<MultiSymbolicExpressionTreeManipulator>().First();
+      alg.StoreAlgorithmInEachRun = false;
+      return alg;
+    }
+
+    private void SampleTrainingData(MersenneTwister rand, ModifiableDataset ds, int rRows,
+      IDataset sourceDs, double[] curTarget, string targetVarName, IEnumerable<int> trainingIndices) {
+      var selectedRows = trainingIndices.SampleRandomWithoutRepetition(rand, rRows).ToArray();
+      int t = 0;
+      object[] srcRow = new object[ds.Columns];
+      var varNames = ds.DoubleVariables.ToArray();
+      foreach (var r in selectedRows) {
+        // take all values from the original dataset
+        for (int c = 0; c < srcRow.Length; c++) {
+          var col = sourceDs.GetReadOnlyDoubleValues(varNames[c]);
+          srcRow[c] = col[r];
+        }
+        ds.ReplaceRow(t, srcRow);
+        // but use the updated target values
+        ds.SetVariableValue(curTarget[r], targetVarName, t);
+        t++;
+      }
+    }
+
+    private static ISymbolicRegressionSolution CreateSymbolicSolution(List<IRegressionModel> models, double nu, IRegressionProblemData problemData) {
+      var symbModels = models.OfType<ISymbolicRegressionModel>();
+      var lowerLimit = symbModels.Min(m => m.LowerEstimationLimit);
+      var upperLimit = symbModels.Max(m => m.UpperEstimationLimit);
+      var interpreter = new SymbolicDataAnalysisExpressionTreeLinearInterpreter();
+      var progRootNode = new ProgramRootSymbol().CreateTreeNode();
+      var startNode = new StartSymbol().CreateTreeNode();
+
+      var addNode = new Addition().CreateTreeNode();
+      var mulNode = new Multiplication().CreateTreeNode();
+      var scaleNode = (ConstantTreeNode)new Constant().CreateTreeNode(); // all models are scaled using the same nu
+      scaleNode.Value = nu;
+
+      foreach (var m in symbModels) {
+        var relevantPart = m.SymbolicExpressionTree.Root.GetSubtree(0).GetSubtree(0); // skip root and start
+        addNode.AddSubtree((ISymbolicExpressionTreeNode)relevantPart.Clone());
+      }
+
+      mulNode.AddSubtree(addNode);
+      mulNode.AddSubtree(scaleNode);
+      startNode.AddSubtree(mulNode);
+      progRootNode.AddSubtree(startNode);
+      var t = new SymbolicExpressionTree(progRootNode);
+      var combinedModel = new SymbolicRegressionModel(problemData.TargetVariable, t, interpreter, lowerLimit, upperLimit);
+      var sol = new SymbolicRegressionSolution(combinedModel, problemData);
+      return sol;
+    }
+
+    private static bool TrySetProblemData(IAlgorithm alg, IRegressionProblemData problemData) {
+      var prob = alg.Problem as IRegressionProblem;
+      // there is already a problem and it is compatible -> just set problem data
+      if (prob != null) {
+        prob.ProblemDataParameter.Value = problemData;
+        return true;
+      } else return false;
+    }
+
+    private static bool TryExecute(IAlgorithm alg, int seed, string regressionAlgorithmResultName, out IRegressionModel model, out IRun run) {
+      model = null;
+      SetSeed(alg, seed);
+      using (var wh = new AutoResetEvent(false)) {
+        Exception ex = null;
+        EventHandler<EventArgs<Exception>> handler = (sender, args) => {
+          ex = args.Value;
+          wh.Set();
+        };
+        EventHandler handler2 = (sender, args) => wh.Set();
+        alg.ExceptionOccurred += handler;
+        alg.Stopped += handler2;
+        try {
+          alg.Prepare();
+          alg.Start();
+          wh.WaitOne();
+
+          if (ex != null) throw new AggregateException(ex);
+          run = alg.Runs.Last();
+          alg.Runs.Clear();
+          var sols = alg.Results.Select(r => r.Value).OfType<IRegressionSolution>();
+          if (!sols.Any()) return false;
+          var sol = sols.First();
+          if (sols.Skip(1).Any()) {
+            // more than one solution => use regressionAlgorithmResult
+            if (alg.Results.ContainsKey(regressionAlgorithmResultName)) {
+              sol = (IRegressionSolution)alg.Results[regressionAlgorithmResultName].Value;
+            }
+          }
+          var symbRegSol = sol as SymbolicRegressionSolution;
+          // only accept symb reg solutions that do not hit the estimation limits
+          // NaN evaluations would not be critical but are problematic if we want to combine all symbolic models into a single symbolic model 
+          if (symbRegSol == null ||
+            (symbRegSol.TrainingLowerEstimationLimitHits == 0 && symbRegSol.TrainingUpperEstimationLimitHits == 0 &&
+             symbRegSol.TestLowerEstimationLimitHits == 0 && symbRegSol.TestUpperEstimationLimitHits == 0) &&
+            symbRegSol.TrainingNaNEvaluations == 0 && symbRegSol.TestNaNEvaluations == 0) {
+            model = sol.Model;
+          }
+        }
+        finally {
+          alg.ExceptionOccurred -= handler;
+          alg.Stopped -= handler2;
+        }
+      }
+      return model != null;
+    }
+
+    private static void SetSeed(IAlgorithm alg, int seed) {
+      // no common interface for algs that use a PRNG -> use naming convention to set seed
+      var paramItem = alg as IParameterizedItem;
+
+      if (paramItem.Parameters.ContainsKey("SetSeedRandomly")) {
+        ((BoolValue)paramItem.Parameters["SetSeedRandomly"].ActualValue).Value = false;
+        ((IntValue)paramItem.Parameters["Seed"].ActualValue).Value = seed;
+      } else {
+        throw new ArgumentException("Base learner does not have a seed parameter (algorithm {0})", alg.Name);
+      }
+
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceConst.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceConst.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceConst.cs	(revision 17041)
@@ -0,0 +1,97 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("B359EC1D-0EAE-4800-9502-133A280CE8B0")]
+  [Item(Name = "CovarianceConst",
+    Description = "Constant covariance function for Gaussian processes.")]
+  public sealed class CovarianceConst : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+    private bool HasFixedScaleParameter {
+      get { return ScaleParameter.Value != null; }
+    }
+    [StorableConstructor]
+    private CovarianceConst(StorableConstructorFlag _) : base(_) {
+    }
+
+    private CovarianceConst(CovarianceConst original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovarianceConst()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale of the constant covariance function."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceConst(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return HasFixedScaleParameter ? 0 : 1;
+    }
+
+    public void SetParameter(double[] p) {
+      double scale;
+      GetParameterValues(p, out scale);
+      ScaleParameter.Value = new DoubleValue(scale);
+    }
+
+    private void GetParameterValues(double[] p, out double scale) {
+      int c = 0;
+      // gather parameter values
+      if (HasFixedScaleParameter) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[c]);
+        c++;
+      }
+      if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceConst", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      double scale;
+      GetParameterValues(p, out scale);
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => scale;
+      cov.CrossCovariance = (x, xt, i, j) => scale;
+      if (HasFixedScaleParameter) {
+        cov.CovarianceGradient = (x, i, j) => new double[0];
+      } else {
+        cov.CovarianceGradient = (x, i, j) => new[] { 2.0 * scale };
+      }
+      return cov;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinear.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinear.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinear.cs	(revision 17041)
@@ -0,0 +1,62 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("5C4AB3C8-48D5-4B6E-830A-89AE2A363C64")]
+  [Item(Name = "CovarianceLinear", Description = "Linear covariance function for Gaussian processes.")]
+  public sealed class CovarianceLinear : Item, ICovarianceFunction {
+    [StorableConstructor]
+    private CovarianceLinear(StorableConstructorFlag _) : base(_) { }
+    private CovarianceLinear(CovarianceLinear original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public CovarianceLinear()
+      : base() {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceLinear(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return 0;
+    }
+
+    public void SetParameter(double[] p) {
+      if (p.Length > 0) throw new ArgumentException("No parameters are allowed for the linear covariance function.");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      if (p.Length > 0) throw new ArgumentException("No parameters are allowed for the linear covariance function.");
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => Util.ScalarProd(x, i, j, columnIndices, 1.0);
+      cov.CrossCovariance = (x, xt, i, j) => Util.ScalarProd(x, i, xt, j, columnIndices, 1.0);
+      cov.CovarianceGradient = (x, i, j) => new double[0];
+      return cov;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinearArd.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinearArd.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinearArd.cs	(revision 17041)
@@ -0,0 +1,109 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("A704FDF2-0566-4AB5-B752-D713831B016C")]
+  [Item(Name = "CovarianceLinearArd",
+    Description = "Linear covariance function with automatic relevance determination for Gaussian processes.")]
+  public sealed class CovarianceLinearArd : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleArray> InverseLengthParameter {
+      get { return (IValueParameter<DoubleArray>)Parameters["InverseLength"]; }
+    }
+    private bool HasFixedInverseLengthParameter {
+      get { return InverseLengthParameter.Value != null; }
+    }
+
+    [StorableConstructor]
+    private CovarianceLinearArd(StorableConstructorFlag _) : base(_) { }
+    private CovarianceLinearArd(CovarianceLinearArd original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public CovarianceLinearArd()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleArray>("InverseLength",
+                                                             "The inverse length parameter for ARD."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceLinearArd(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      if (HasFixedInverseLengthParameter)
+        return 0;
+      else
+        return numberOfVariables;
+    }
+
+    public void SetParameter(double[] p) {
+      double[] inverseLength;
+      GetParameterValues(p, out inverseLength);
+      InverseLengthParameter.Value = new DoubleArray(inverseLength);
+    }
+
+    private void GetParameterValues(double[] p, out double[] inverseLength) {
+      // gather parameter values
+      if (HasFixedInverseLengthParameter) {
+        inverseLength = InverseLengthParameter.Value.ToArray();
+      } else {
+        inverseLength = p.Select(e => 1.0 / Math.Exp(e)).ToArray();
+      }
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      double[] inverseLength;
+      GetParameterValues(p, out inverseLength);
+      var fixedInverseLength = HasFixedInverseLengthParameter;
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => Util.ScalarProd(x, i, j, inverseLength, columnIndices);
+      cov.CrossCovariance = (x, xt, i, j) => Util.ScalarProd(x, i, xt, j, inverseLength, columnIndices);
+      if (fixedInverseLength)
+        cov.CovarianceGradient = (x, i, j) => new double[0];
+      else
+        cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, inverseLength, columnIndices);
+      return cov;
+    }
+
+    private static IList<double> GetGradient(double[,] x, int i, int j, double[] inverseLength, int[] columnIndices) {
+      int k = 0;
+      var g = new List<double>(columnIndices.Length);
+      for (int c = 0; c < columnIndices.Length; c++) {
+        var columnIndex = columnIndices[c];
+        g.Add(-2.0 * x[i, columnIndex] * x[j, columnIndex] * inverseLength[k] * inverseLength[k]);
+        k++;
+      }
+      return g;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMask.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMask.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMask.cs	(revision 17041)
@@ -0,0 +1,78 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("21D5F261-B9DF-4DC4-AF42-7B223C45840F")]
+  [Item(Name = "CovarianceMask",
+    Description = "Masking covariance function for dimension selection can be used to apply a covariance function only on certain input dimensions.")]
+  public sealed class CovarianceMask : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<IntArray> SelectedDimensionsParameter {
+      get { return (IValueParameter<IntArray>)Parameters["SelectedDimensions"]; }
+    }
+    public IValueParameter<ICovarianceFunction> CovarianceFunctionParameter {
+      get { return (IValueParameter<ICovarianceFunction>)Parameters["CovarianceFunction"]; }
+    }
+
+    [StorableConstructor]
+    private CovarianceMask(StorableConstructorFlag _) : base(_) {
+    }
+
+    private CovarianceMask(CovarianceMask original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovarianceMask()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<IntArray>("SelectedDimensions", "The dimensions on which the specified covariance function should be applied to."));
+      Parameters.Add(new ValueParameter<ICovarianceFunction>("CovarianceFunction", "The covariance function that should be scaled.", new CovarianceSquaredExponentialIso()));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceMask(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      if (SelectedDimensionsParameter.Value == null) return CovarianceFunctionParameter.Value.GetNumberOfParameters(numberOfVariables);
+      else return CovarianceFunctionParameter.Value.GetNumberOfParameters(SelectedDimensionsParameter.Value.Length);
+    }
+
+    public void SetParameter(double[] p) {
+      CovarianceFunctionParameter.Value.SetParameter(p);
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      var cov = CovarianceFunctionParameter.Value;
+      var selectedDimensions = SelectedDimensionsParameter.Value;
+
+      return cov.GetParameterizedCovarianceFunction(p, selectedDimensions.Intersect(columnIndices).ToArray());
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMaternIso.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMaternIso.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMaternIso.cs	(revision 17041)
@@ -0,0 +1,168 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("D251400A-4DCA-4500-9738-CE3B7BF96B0D")]
+  [Item(Name = "CovarianceMaternIso",
+    Description = "Matern covariance function for Gaussian processes.")]
+  public sealed class CovarianceMaternIso : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> InverseLengthParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["InverseLength"]; }
+    }
+
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+
+    public IConstrainedValueParameter<IntValue> DParameter {
+      get { return (IConstrainedValueParameter<IntValue>)Parameters["D"]; }
+    }
+    private bool HasFixedScaleParameter {
+      get { return ScaleParameter.Value != null; }
+    }
+    private bool HasFixedInverseLengthParameter {
+      get { return InverseLengthParameter.Value != null; }
+    }
+
+    [StorableConstructor]
+    private CovarianceMaternIso(StorableConstructorFlag _) : base(_) {
+    }
+
+    private CovarianceMaternIso(CovarianceMaternIso original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovarianceMaternIso()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("InverseLength", "The inverse length parameter of the isometric Matern covariance function."));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale parameter of the isometric Matern covariance function."));
+      var validDValues = new ItemSet<IntValue>();
+      validDValues.Add((IntValue)new IntValue(1).AsReadOnly());
+      validDValues.Add((IntValue)new IntValue(3).AsReadOnly());
+      validDValues.Add((IntValue)new IntValue(5).AsReadOnly());
+      Parameters.Add(new ConstrainedValueParameter<IntValue>("D", "The d parameter (allowed values: 1, 3, or 5) of the isometric Matern covariance function.", validDValues, validDValues.First()));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceMaternIso(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return
+        (HasFixedInverseLengthParameter ? 0 : 1) +
+        (HasFixedScaleParameter ? 0 : 1);
+    }
+
+    public void SetParameter(double[] p) {
+      double inverseLength, scale;
+      GetParameterValues(p, out scale, out inverseLength);
+      InverseLengthParameter.Value = new DoubleValue(inverseLength);
+      ScaleParameter.Value = new DoubleValue(scale);
+    }
+
+    private void GetParameterValues(double[] p, out double scale, out double inverseLength) {
+      // gather parameter values
+      int c = 0;
+      if (HasFixedInverseLengthParameter) {
+        inverseLength = InverseLengthParameter.Value.Value;
+      } else {
+        inverseLength = 1.0 / Math.Exp(p[c]);
+        c++;
+      }
+
+      if (HasFixedScaleParameter) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[c]);
+        c++;
+      }
+      if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceMaternIso", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      double inverseLength, scale;
+      int d = DParameter.Value.Value;
+      GetParameterValues(p, out scale, out inverseLength);
+      var fixedInverseLength = HasFixedInverseLengthParameter;
+      var fixedScale = HasFixedScaleParameter;
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => {
+        double dist = i == j
+                       ? 0.0
+                       : Math.Sqrt(Util.SqrDist(x, i, j, columnIndices, Math.Sqrt(d) * inverseLength));
+        return scale * m(d, dist);
+      };
+      cov.CrossCovariance = (x, xt, i, j) => {
+        double dist = Math.Sqrt(Util.SqrDist(x, i, xt, j, columnIndices, Math.Sqrt(d) * inverseLength));
+        return scale * m(d, dist);
+      };
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, d, scale, inverseLength, columnIndices, fixedInverseLength, fixedScale);
+      return cov;
+    }
+
+    private static double m(int d, double t) {
+      double f;
+      switch (d) {
+        case 1: { f = 1; break; }
+        case 3: { f = 1 + t; break; }
+        case 5: { f = 1 + t * (1 + t / 3.0); break; }
+        default: throw new InvalidOperationException();
+      }
+      return f * Math.Exp(-t);
+    }
+
+    private static double dm(int d, double t) {
+      double df;
+      switch (d) {
+        case 1: { df = 1; break; }
+        case 3: { df = t; break; }
+        case 5: { df = t * (1 + t) / 3.0; break; }
+        default: throw new InvalidOperationException();
+      }
+      return df * t * Math.Exp(-t);
+    }
+
+    private static IList<double> GetGradient(double[,] x, int i, int j, int d, double scale, double inverseLength, int[] columnIndices,
+      bool fixedInverseLength, bool fixedScale) {
+      double dist = i == j
+                   ? 0.0
+                   : Math.Sqrt(Util.SqrDist(x, i, j, columnIndices, Math.Sqrt(d) * inverseLength));
+
+      var g = new List<double>(2);
+      if (!fixedInverseLength) g.Add(scale * dm(d, dist));
+      if (!fixedScale) g.Add(2 * scale * m(d, dist));
+      return g;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNeuralNetwork.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNeuralNetwork.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNeuralNetwork.cs	(revision 17041)
@@ -0,0 +1,161 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("F60E0A63-0107-44E3-920B-BB5B09E9DDDF")]
+  [Item(Name = "CovarianceNeuralNetwork",
+    Description = "Neural network covariance function for Gaussian processes.")]
+  public sealed class CovarianceNeuralNetwork : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+
+    public IValueParameter<DoubleValue> LengthParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Length"]; }
+    }
+    private bool HasFixedScaleParameter {
+      get { return ScaleParameter.Value != null; }
+    }
+    private bool HasFixedLengthParameter {
+      get { return LengthParameter.Value != null; }
+    }
+
+    [StorableConstructor]
+    private CovarianceNeuralNetwork(StorableConstructorFlag _) : base(_) {
+    }
+
+    private CovarianceNeuralNetwork(CovarianceNeuralNetwork original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovarianceNeuralNetwork()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale parameter."));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Length", "The length parameter."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceNeuralNetwork(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return
+        (HasFixedScaleParameter ? 0 : 1) +
+        (HasFixedLengthParameter ? 0 : 1);
+    }
+
+    public void SetParameter(double[] p) {
+      double scale, length;
+      GetParameterValues(p, out scale, out length);
+      ScaleParameter.Value = new DoubleValue(scale);
+      LengthParameter.Value = new DoubleValue(length);
+    }
+
+
+    private void GetParameterValues(double[] p, out double scale, out double length) {
+      // gather parameter values
+      int c = 0;
+      if (HasFixedLengthParameter) {
+        length = LengthParameter.Value.Value;
+      } else {
+        length = Math.Exp(2 * p[c]);
+        c++;
+      }
+
+      if (HasFixedScaleParameter) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[c]);
+        c++;
+      }
+      if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceNeuralNetwork", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      double length, scale;
+      GetParameterValues(p, out scale, out length);
+      var fixedLength = HasFixedLengthParameter;
+      var fixedScale = HasFixedScaleParameter;
+
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => {
+        double sx = 1.0;
+        double s1 = 1.0;
+        double s2 = 1.0;
+        for (int c = 0; c < columnIndices.Length; c++) {
+          var col = columnIndices[c];
+          sx += x[i, col] * x[j, col];
+          s1 += x[i, col] * x[i, col];
+          s2 += x[j, col] * x[j, col];
+        }
+
+        return (scale * Math.Asin(sx / (Math.Sqrt((length + s1) * (length + s2)))));
+      };
+      cov.CrossCovariance = (x, xt, i, j) => {
+        double sx = 1.0;
+        double s1 = 1.0;
+        double s2 = 1.0;
+        for (int c = 0; c < columnIndices.Length; c++) {
+          var col = columnIndices[c];
+          sx += x[i, col] * xt[j, col];
+          s1 += x[i, col] * x[i, col];
+          s2 += xt[j, col] * xt[j, col];
+        }
+
+        return (scale * Math.Asin(sx / (Math.Sqrt((length + s1) * (length + s2)))));
+      };
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, length, scale, columnIndices, fixedLength, fixedScale);
+      return cov;
+    }
+
+    // order of returned gradients must match the order in GetParameterValues!
+    private static IList<double> GetGradient(double[,] x, int i, int j, double length, double scale, int[] columnIndices,
+      bool fixedLength, bool fixedScale) {
+      double sx = 1.0;
+      double s1 = 1.0;
+      double s2 = 1.0;
+      for (int c = 0; c < columnIndices.Length; c++) {
+        var col = columnIndices[c];
+        sx += x[i, col] * x[j, col];
+        s1 += x[i, col] * x[i, col];
+        s2 += x[j, col] * x[j, col];
+      }
+      var h = (length + s1) * (length + s2);
+      var f = sx / Math.Sqrt(h);
+
+      var g = new List<double>(2);
+      if (!fixedLength) g.Add(-scale / Math.Sqrt(1.0 - f * f) * ((length * sx * (2.0 * length + s1 + s2)) / Math.Pow(h, 3.0 / 2.0)));
+      if (!fixedScale) g.Add(2.0 * scale * Math.Asin(f));
+      return g;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNoise.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNoise.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNoise.cs	(revision 17041)
@@ -0,0 +1,98 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("C6AEEC11-1F8D-40D1-8D8A-DCCCE886E46C")]
+  [Item(Name = "CovarianceNoise",
+    Description = "Noise covariance function for Gaussian processes.")]
+  public sealed class CovarianceNoise : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+    private bool HasFixedScaleParameter {
+      get { return ScaleParameter.Value != null; }
+    }
+
+    [StorableConstructor]
+    private CovarianceNoise(StorableConstructorFlag _) : base(_) {
+    }
+
+    private CovarianceNoise(CovarianceNoise original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovarianceNoise()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale of noise."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceNoise(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return HasFixedScaleParameter ? 0 : 1;
+    }
+
+    public void SetParameter(double[] p) {
+      double scale;
+      GetParameterValues(p, out scale);
+      ScaleParameter.Value = new DoubleValue(scale);
+    }
+
+    private void GetParameterValues(double[] p, out double scale) {
+      int c = 0;
+      // gather parameter values
+      if (HasFixedScaleParameter) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[c]);
+        c++;
+      }
+      if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceNoise", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      double scale;
+      GetParameterValues(p, out scale);
+      var fixedScale = HasFixedScaleParameter;
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => i == j ? scale : 0.0;
+      cov.CrossCovariance = (x, xt, i, j) => Util.SqrDist(x, i, xt, j, columnIndices, 1.0) < 1e-9 ? scale : 0.0;
+      if (fixedScale)
+        cov.CovarianceGradient = (x, i, j) => new double[0];
+      else
+        cov.CovarianceGradient = (x, i, j) => new double[1] { i == j ? 2.0 * scale : 0.0 };
+      return cov;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePeriodic.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePeriodic.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePeriodic.cs	(revision 17041)
@@ -0,0 +1,167 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("198B5472-6CAA-4C39-BB1F-6EC16CB7801B")]
+  [Item(Name = "CovariancePeriodic", Description = "Periodic covariance function for Gaussian processes.")]
+  public sealed class CovariancePeriodic : ParameterizedNamedItem, ICovarianceFunction {
+
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+
+    public IValueParameter<DoubleValue> InverseLengthParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["InverseLength"]; }
+    }
+
+    public IValueParameter<DoubleValue> PeriodParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Period"]; }
+    }
+
+    private bool HasFixedScaleParameter {
+      get { return ScaleParameter.Value != null; }
+    }
+    private bool HasFixedInverseLengthParameter {
+      get { return InverseLengthParameter.Value != null; }
+    }
+    private bool HasFixedPeriodParameter {
+      get { return PeriodParameter.Value != null; }
+    }
+
+
+    [StorableConstructor]
+    private CovariancePeriodic(StorableConstructorFlag _) : base(_) { }
+    private CovariancePeriodic(CovariancePeriodic original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovariancePeriodic()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale of the periodic covariance function."));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("InverseLength", "The inverse length parameter for the periodic covariance function."));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Period", "The period parameter for the periodic covariance function."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovariancePeriodic(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return (HasFixedScaleParameter ? 0 : 1) +
+       (HasFixedPeriodParameter ? 0 : 1) +
+       (HasFixedInverseLengthParameter ? 0 : 1);
+    }
+
+    public void SetParameter(double[] p) {
+      double scale, inverseLength, period;
+      GetParameterValues(p, out scale, out period, out inverseLength);
+      ScaleParameter.Value = new DoubleValue(scale);
+      PeriodParameter.Value = new DoubleValue(period);
+      InverseLengthParameter.Value = new DoubleValue(inverseLength);
+    }
+
+
+    private void GetParameterValues(double[]
+      p, out double scale, out double period, out double inverseLength) {
+      // gather parameter values
+      int c = 0;
+      if (HasFixedInverseLengthParameter) {
+        inverseLength = InverseLengthParameter.Value.Value;
+      } else {
+        inverseLength = 1.0 / Math.Exp(p[c]);
+        c++;
+      }
+      if (HasFixedPeriodParameter) {
+        period = PeriodParameter.Value.Value;
+      } else {
+        period = Math.Exp(p[c]);
+        c++;
+      }
+      if (HasFixedScaleParameter) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[c]);
+        c++;
+      }
+      if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovariancePeriodic", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      double inverseLength, period, scale;
+      GetParameterValues(p, out scale, out period, out inverseLength);
+      var fixedInverseLength = HasFixedInverseLengthParameter;
+      var fixedPeriod = HasFixedPeriodParameter;
+      var fixedScale = HasFixedScaleParameter;
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => {
+        double k = i == j ? 0.0 : GetDistance(x, x, i, j, columnIndices);
+        k = Math.PI * k / period;
+        k = Math.Sin(k) * inverseLength;
+        k = k * k;
+
+        return scale * Math.Exp(-2.0 * k);
+      };
+      cov.CrossCovariance = (x, xt, i, j) => {
+        double k = GetDistance(x, xt, i, j, columnIndices);
+        k = Math.PI * k / period;
+        k = Math.Sin(k) * inverseLength;
+        k = k * k;
+
+        return scale * Math.Exp(-2.0 * k);
+      };
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, columnIndices, scale, period, inverseLength, fixedInverseLength, fixedPeriod, fixedScale);
+      return cov;
+    }
+
+    private static IList<double> GetGradient(double[,] x, int i, int j, int[] columnIndices, double scale, double period, double inverseLength,
+      bool fixedInverseLength, bool fixedPeriod, bool fixedScale) {
+      double k = i == j ? 0.0 : Math.PI * GetDistance(x, x, i, j, columnIndices) / period;
+      double gradient = Math.Sin(k) * inverseLength;
+      gradient *= gradient;
+      var g = new List<double>(3);
+      if (!fixedInverseLength) 
+        g.Add(4.0 * scale * Math.Exp(-2.0 * gradient) * gradient);
+      if (!fixedPeriod) {
+        double r = Math.Sin(k) * inverseLength;
+        g.Add(2.0 * k * scale * Math.Exp(-2 * r * r) * Math.Sin(2 * k) * inverseLength * inverseLength);
+      }
+      if (!fixedScale)
+        g.Add(2.0 * scale * Math.Exp(-2 * gradient));
+      return g;
+    }
+
+    private static double GetDistance(double[,] x, double[,] xt, int i, int j, int[] columnIndices) {
+      return Math.Sqrt(Util.SqrDist(x, i, xt, j, columnIndices, 1));
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePiecewisePolynomial.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePiecewisePolynomial.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePiecewisePolynomial.cs	(revision 17041)
@@ -0,0 +1,169 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("62C25AD5-F41F-4CC2-B589-A92CCEE7AC88")]
+  [Item(Name = "CovariancePiecewisePolynomial",
+    Description = "Piecewise polynomial covariance function with compact support for Gaussian processes.")]
+  public sealed class CovariancePiecewisePolynomial : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> LengthParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Length"]; }
+    }
+
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+
+    public IConstrainedValueParameter<IntValue> VParameter {
+      get { return (IConstrainedValueParameter<IntValue>)Parameters["V"]; }
+    }
+    private bool HasFixedLengthParameter {
+      get { return LengthParameter.Value != null; }
+    }
+    private bool HasFixedScaleParameter {
+      get { return ScaleParameter.Value != null; }
+    }
+
+    [StorableConstructor]
+    private CovariancePiecewisePolynomial(StorableConstructorFlag _) : base(_) {
+    }
+
+    private CovariancePiecewisePolynomial(CovariancePiecewisePolynomial original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovariancePiecewisePolynomial()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Length", "The length parameter of the isometric piecewise polynomial covariance function."));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale parameter of the piecewise polynomial covariance function."));
+
+      var validValues = new ItemSet<IntValue>(new IntValue[] {
+        (IntValue)(new IntValue().AsReadOnly()),
+        (IntValue)(new IntValue(1).AsReadOnly()),
+        (IntValue)(new IntValue(2).AsReadOnly()),
+        (IntValue)(new IntValue(3).AsReadOnly()) });
+      Parameters.Add(new ConstrainedValueParameter<IntValue>("V", "The v parameter of the piecewise polynomial function (allowed values 0, 1, 2, 3).", validValues, validValues.First()));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovariancePiecewisePolynomial(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return
+        (HasFixedLengthParameter ? 0 : 1) +
+        (HasFixedScaleParameter ? 0 : 1);
+    }
+
+    public void SetParameter(double[] p) {
+      double @const, scale;
+      GetParameterValues(p, out @const, out scale);
+      LengthParameter.Value = new DoubleValue(@const);
+      ScaleParameter.Value = new DoubleValue(scale);
+    }
+
+    private void GetParameterValues(double[] p, out double length, out double scale) {
+      // gather parameter values
+      int n = 0;
+      if (HasFixedLengthParameter) {
+        length = LengthParameter.Value.Value;
+      } else {
+        length = Math.Exp(p[n]);
+        n++;
+      }
+
+      if (HasFixedScaleParameter) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[n]);
+        n++;
+      }
+      if (p.Length != n) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovariancePiecewisePolynomial", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      double length, scale;
+      int v = VParameter.Value.Value;
+      GetParameterValues(p, out length, out scale);
+      var fixedLength = HasFixedLengthParameter;
+      var fixedScale = HasFixedScaleParameter;
+      int exp = (int)Math.Floor(columnIndices.Count() / 2.0) + v + 1;
+
+      Func<double, double> f;
+      Func<double, double> df;
+      switch (v) {
+        case 0:
+          f = (r) => 1.0;
+          df = (r) => 0.0;
+          break;
+        case 1:
+          f = (r) => 1 + (exp + 1) * r;
+          df = (r) => exp + 1;
+          break;
+        case 2:
+          f = (r) => 1 + (exp + 2) * r + (exp * exp + 4.0 * exp + 3) / 3.0 * r * r;
+          df = (r) => (exp + 2) + 2 * (exp * exp + 4.0 * exp + 3) / 3.0 * r;
+          break;
+        case 3:
+          f = (r) => 1 + (exp + 3) * r + (6.0 * exp * exp + 36 * exp + 45) / 15.0 * r * r +
+                     (exp * exp * exp + 9 * exp * exp + 23 * exp + 45) / 15.0 * r * r * r;
+          df = (r) => (exp + 3) + 2 * (6.0 * exp * exp + 36 * exp + 45) / 15.0 * r +
+                      (exp * exp * exp + 9 * exp * exp + 23 * exp + 45) / 5.0 * r * r;
+          break;
+        default: throw new ArgumentException();
+      }
+
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => {
+        double k = Math.Sqrt(Util.SqrDist(x, i, x, j, columnIndices, 1.0 / length));
+        return scale * Math.Pow(Math.Max(1 - k, 0), exp + v) * f(k);
+      };
+      cov.CrossCovariance = (x, xt, i, j) => {
+        double k = Math.Sqrt(Util.SqrDist(x, i, xt, j, columnIndices, 1.0 / length));
+        return scale * Math.Pow(Math.Max(1 - k, 0), exp + v) * f(k);
+      };
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, length, scale, v, exp, f, df, columnIndices, fixedLength, fixedScale);
+      return cov;
+    }
+
+    private static IList<double> GetGradient(double[,] x, int i, int j, double length, double scale, int v, double exp, Func<double, double> f, Func<double, double> df, int[] columnIndices,
+      bool fixedLength, bool fixedScale) {
+      double k = Math.Sqrt(Util.SqrDist(x, i, x, j, columnIndices, 1.0 / length));
+      var g = new List<double>(2);
+      if (!fixedLength) g.Add(scale * Math.Pow(Math.Max(1.0 - k, 0), exp + v - 1) * k * ((exp + v) * f(k) - Math.Max(1 - k, 0) * df(k)));
+      if (!fixedScale) g.Add(2.0 * scale * Math.Pow(Math.Max(1 - k, 0), exp + v) * f(k));
+      return g;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePolynomial.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePolynomial.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePolynomial.cs	(revision 17041)
@@ -0,0 +1,131 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("BD6DF0C6-07A2-44CE-8EDB-92561505EF6E")]
+  [Item(Name = "CovariancePolynomial",
+    Description = "Polynomial covariance function for Gaussian processes.")]
+  public sealed class CovariancePolynomial : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> ConstParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Const"]; }
+    }
+
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+
+    public IValueParameter<IntValue> DegreeParameter {
+      get { return (IValueParameter<IntValue>)Parameters["Degree"]; }
+    }
+    private bool HasFixedConstParameter {
+      get { return ConstParameter.Value != null; }
+    }
+    private bool HasFixedScaleParameter {
+      get { return ScaleParameter.Value != null; }
+    }
+
+    [StorableConstructor]
+    private CovariancePolynomial(StorableConstructorFlag _) : base(_) {
+    }
+
+    private CovariancePolynomial(CovariancePolynomial original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovariancePolynomial()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Const", "Additive constant in the polymomial."));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale parameter of the polynomial covariance function."));
+      Parameters.Add(new ValueParameter<IntValue>("Degree", "The degree of the polynomial (only non-zero positive values allowed).", new IntValue(2)));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovariancePolynomial(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return
+        (HasFixedConstParameter ? 0 : 1) +
+        (HasFixedScaleParameter ? 0 : 1);
+    }
+
+    public void SetParameter(double[] p) {
+      double @const, scale;
+      GetParameterValues(p, out @const, out scale);
+      ConstParameter.Value = new DoubleValue(@const);
+      ScaleParameter.Value = new DoubleValue(scale);
+    }
+
+    private void GetParameterValues(double[] p, out double @const, out double scale) {
+      // gather parameter values
+      int n = 0;
+      if (HasFixedConstParameter) {
+        @const = ConstParameter.Value.Value;
+      } else {
+        @const = Math.Exp(p[n]);
+        n++;
+      }
+
+      if (HasFixedScaleParameter) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[n]);
+        n++;
+      }
+      if (p.Length != n) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovariancePolynomial", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      double @const, scale;
+      int degree = DegreeParameter.Value.Value;
+      if (degree <= 0) throw new ArgumentException("The degree parameter for CovariancePolynomial must be greater than zero.");
+      GetParameterValues(p, out @const, out scale);
+      var fixedConst = HasFixedConstParameter;
+      var fixedScale = HasFixedScaleParameter;
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => scale * Math.Pow(@const + Util.ScalarProd(x, i, j, columnIndices, 1.0), degree);
+      cov.CrossCovariance = (x, xt, i, j) => scale * Math.Pow(@const + Util.ScalarProd(x, i, xt, j, columnIndices, 1.0), degree);
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, @const, scale, degree, columnIndices, fixedConst, fixedScale);
+      return cov;
+    }
+
+    private static IList<double> GetGradient(double[,] x, int i, int j, double c, double scale, int degree, int[] columnIndices,
+      bool fixedConst, bool fixedScale) {
+      double s = Util.ScalarProd(x, i, j, columnIndices, 1.0);
+      var g = new List<double>(2);
+      if (!fixedConst) g.Add(c * degree * scale * Math.Pow(c + s, degree - 1));
+      if (!fixedScale) g.Add(2 * scale * Math.Pow(c + s, degree));
+      return g;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceProduct.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceProduct.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceProduct.cs	(revision 17041)
@@ -0,0 +1,107 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("C65B207C-2E95-4764-8698-D0B1D1EECCF1")]
+  [Item(Name = "CovarianceProduct",
+    Description = "Product covariance function for Gaussian processes.")]
+  public sealed class CovarianceProduct : Item, ICovarianceFunction {
+    [Storable]
+    private ItemList<ICovarianceFunction> factors;
+
+    [Storable]
+    private int numberOfVariables;
+    public ItemList<ICovarianceFunction> Factors {
+      get { return factors; }
+    }
+
+    [StorableConstructor]
+    private CovarianceProduct(StorableConstructorFlag _) : base(_) {
+    }
+
+    private CovarianceProduct(CovarianceProduct original, Cloner cloner)
+      : base(original, cloner) {
+      this.factors = cloner.Clone(original.factors);
+      this.numberOfVariables = original.numberOfVariables;
+    }
+
+    public CovarianceProduct()
+      : base() {
+      this.factors = new ItemList<ICovarianceFunction>();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceProduct(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      this.numberOfVariables = numberOfVariables;
+      return factors.Select(f => f.GetNumberOfParameters(numberOfVariables)).Sum();
+    }
+
+    public void SetParameter(double[] p) {
+      int offset = 0;
+      foreach (var f in factors) {
+        var numberOfParameters = f.GetNumberOfParameters(numberOfVariables);
+        f.SetParameter(p.Skip(offset).Take(numberOfParameters).ToArray());
+        offset += numberOfParameters;
+      }
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      if (factors.Count == 0) throw new ArgumentException("at least one factor is necessary for the product covariance function.");
+      var functions = new List<ParameterizedCovarianceFunction>();
+      foreach (var f in factors) {
+        int numberOfParameters = f.GetNumberOfParameters(numberOfVariables);
+        functions.Add(f.GetParameterizedCovarianceFunction(p.Take(numberOfParameters).ToArray(), columnIndices));
+        p = p.Skip(numberOfParameters).ToArray();
+      }
+
+
+      var product = new ParameterizedCovarianceFunction();
+      product.Covariance = (x, i, j) => functions.Select(e => e.Covariance(x, i, j)).Aggregate((a, b) => a * b);
+      product.CrossCovariance = (x, xt, i, j) => functions.Select(e => e.CrossCovariance(x, xt, i, j)).Aggregate((a, b) => a * b);
+      product.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, functions);
+      return product;
+    }
+
+    public static IList<double> GetGradient(double[,] x, int i, int j, List<ParameterizedCovarianceFunction> factorFunctions) {
+      var covariances = factorFunctions.Select(f => f.Covariance(x, i, j)).ToArray();
+      var gr = new List<double>();
+      for (int ii = 0; ii < factorFunctions.Count; ii++) {
+        foreach (var g in factorFunctions[ii].CovarianceGradient(x, i, j)) {
+          double res = g;
+          for (int jj = 0; jj < covariances.Length; jj++)
+            if (ii != jj) res *= covariances[jj];
+          gr.Add(res);
+        }
+      }
+      return gr;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticArd.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticArd.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticArd.cs	(revision 17041)
@@ -0,0 +1,166 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("25439318-762C-4ED6-BAE7-E53860598A01")]
+  [Item(Name = "CovarianceRationalQuadraticArd",
+    Description = "Rational quadratic covariance function with automatic relevance determination for Gaussian processes.")]
+  public sealed class CovarianceRationalQuadraticArd : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+
+    public IValueParameter<DoubleArray> InverseLengthParameter {
+      get { return (IValueParameter<DoubleArray>)Parameters["InverseLength"]; }
+    }
+
+    public IValueParameter<DoubleValue> ShapeParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Shape"]; }
+    }
+    private bool HasFixedScaleParameter {
+      get { return ScaleParameter.Value != null; }
+    }
+    private bool HasFixedInverseLengthParameter {
+      get { return InverseLengthParameter.Value != null; }
+    }
+    private bool HasFixedShapeParameter {
+      get { return ShapeParameter.Value != null; }
+    }
+
+    [StorableConstructor]
+    private CovarianceRationalQuadraticArd(StorableConstructorFlag _) : base(_) {
+    }
+
+    private CovarianceRationalQuadraticArd(CovarianceRationalQuadraticArd original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovarianceRationalQuadraticArd()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale parameter of the rational quadratic covariance function with ARD."));
+      Parameters.Add(new OptionalValueParameter<DoubleArray>("InverseLength", "The inverse length parameter for automatic relevance determination."));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Shape", "The shape parameter (alpha) of the rational quadratic covariance function with ARD."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceRationalQuadraticArd(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return
+        (HasFixedScaleParameter ? 0 : 1) +
+        (HasFixedShapeParameter ? 0 : 1) +
+        (HasFixedInverseLengthParameter ? 0 : numberOfVariables);
+    }
+
+    public void SetParameter(double[] p) {
+      double scale, shape;
+      double[] inverseLength;
+      GetParameterValues(p, out scale, out shape, out inverseLength);
+      ScaleParameter.Value = new DoubleValue(scale);
+      ShapeParameter.Value = new DoubleValue(shape);
+      InverseLengthParameter.Value = new DoubleArray(inverseLength);
+    }
+
+    private void GetParameterValues(double[] p, out double scale, out double shape, out double[] inverseLength) {
+      int c = 0;
+      // gather parameter values
+      if (HasFixedInverseLengthParameter) {
+        inverseLength = InverseLengthParameter.Value.ToArray();
+      } else {
+        int length = p.Length;
+        if (!HasFixedScaleParameter) length--;
+        if (!HasFixedShapeParameter) length--;
+        inverseLength = p.Select(e => 1.0 / Math.Exp(e)).Take(length).ToArray();
+        c += inverseLength.Length;
+      }
+      if (HasFixedScaleParameter) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[c]);
+        c++;
+      }
+      if (HasFixedShapeParameter) {
+        shape = ShapeParameter.Value.Value;
+      } else {
+        shape = Math.Exp(p[c]);
+        c++;
+      }
+      if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceRationalQuadraticArd", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      double scale, shape;
+      double[] inverseLength;
+      GetParameterValues(p, out scale, out shape, out inverseLength);
+      var fixedInverseLength = HasFixedInverseLengthParameter;
+      var fixedScale = HasFixedScaleParameter;
+      var fixedShape = HasFixedShapeParameter;
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => {
+        double d = i == j
+                    ? 0.0
+                    : Util.SqrDist(x, i, j, inverseLength, columnIndices);
+        return scale * Math.Pow(1 + 0.5 * d / shape, -shape);
+      };
+      cov.CrossCovariance = (x, xt, i, j) => {
+        double d = Util.SqrDist(x, i, xt, j, inverseLength, columnIndices);
+        return scale * Math.Pow(1 + 0.5 * d / shape, -shape);
+      };
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, columnIndices, scale, shape, inverseLength, fixedInverseLength, fixedScale, fixedShape);
+      return cov;
+    }
+
+    private static IList<double> GetGradient(double[,] x, int i, int j, int[] columnIndices, double scale, double shape, double[] inverseLength,
+      bool fixedInverseLength, bool fixedScale, bool fixedShape) {
+      double d = i == j
+                   ? 0.0
+                   : Util.SqrDist(x, i, j, inverseLength, columnIndices);
+      double b = 1 + 0.5 * d / shape;
+      int k = 0;
+      var g = new List<double>(columnIndices.Length + 2);
+      if (!fixedInverseLength) {
+        foreach (var columnIndex in columnIndices) {
+          g.Add(
+            scale * Math.Pow(b, -shape - 1) *
+            Util.SqrDist(x[i, columnIndex] * inverseLength[k], x[j, columnIndex] * inverseLength[k]));
+          k++;
+        }
+      }
+      if (!fixedScale) g.Add(2 * scale * Math.Pow(b, -shape));
+      if (!fixedShape) g.Add(scale * Math.Pow(b, -shape) * (0.5 * d / b - shape * Math.Log(b)));
+      return g;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticIso.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticIso.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticIso.cs	(revision 17041)
@@ -0,0 +1,154 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("358BE57A-13C8-40BB-B344-217984D4EB0F")]
+  [Item(Name = "CovarianceRationalQuadraticIso",
+    Description = "Isotropic rational quadratic covariance function for Gaussian processes.")]
+  public sealed class CovarianceRationalQuadraticIso : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+
+    public IValueParameter<DoubleValue> InverseLengthParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["InverseLength"]; }
+    }
+
+    public IValueParameter<DoubleValue> ShapeParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Shape"]; }
+    }
+
+    private bool HasFixedScaleParameter {
+      get { return ScaleParameter.Value != null; }
+    }
+    private bool HasFixedInverseLengthParameter {
+      get { return InverseLengthParameter.Value != null; }
+    }
+    private bool HasFixedShapeParameter {
+      get { return ShapeParameter.Value != null; }
+    }
+
+
+    [StorableConstructor]
+    private CovarianceRationalQuadraticIso(StorableConstructorFlag _) : base(_) {
+    }
+
+    private CovarianceRationalQuadraticIso(CovarianceRationalQuadraticIso original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovarianceRationalQuadraticIso()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale parameter of the isometric rational quadratic covariance function."));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("InverseLength", "The inverse length parameter of the isometric rational quadratic covariance function."));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Shape", "The shape parameter (alpha) of the isometric rational quadratic covariance function."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceRationalQuadraticIso(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return (HasFixedScaleParameter ? 0 : 1) +
+        (HasFixedShapeParameter ? 0 : 1) +
+        (HasFixedInverseLengthParameter ? 0 : 1);
+    }
+
+    public void SetParameter(double[] p) {
+      double scale, shape, inverseLength;
+      GetParameterValues(p, out scale, out shape, out inverseLength);
+      ScaleParameter.Value = new DoubleValue(scale);
+      ShapeParameter.Value = new DoubleValue(shape);
+      InverseLengthParameter.Value = new DoubleValue(inverseLength);
+    }
+
+    private void GetParameterValues(double[] p, out double scale, out double shape, out double inverseLength) {
+      int c = 0;
+      // gather parameter values
+      if (HasFixedInverseLengthParameter) {
+        inverseLength = InverseLengthParameter.Value.Value;
+      } else {
+        inverseLength = 1.0 / Math.Exp(p[c]);
+        c++;
+      }
+      if (HasFixedScaleParameter) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[c]);
+        c++;
+      }
+      if (HasFixedShapeParameter) {
+        shape = ShapeParameter.Value.Value;
+      } else {
+        shape = Math.Exp(p[c]);
+        c++;
+      }
+      if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceRationalQuadraticIso", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      double scale, shape, inverseLength;
+      GetParameterValues(p, out scale, out shape, out inverseLength);
+      var fixedInverseLength = HasFixedInverseLengthParameter;
+      var fixedScale = HasFixedScaleParameter;
+      var fixedShape = HasFixedShapeParameter;
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => {
+        double d = i == j
+                    ? 0.0
+                    : Util.SqrDist(x, i, j, columnIndices, inverseLength);
+        return scale * Math.Pow(1 + 0.5 * d / shape, -shape);
+      };
+      cov.CrossCovariance = (x, xt, i, j) => {
+        double d = Util.SqrDist(x, i, xt, j, columnIndices, inverseLength);
+        return scale * Math.Pow(1 + 0.5 * d / shape, -shape);
+      };
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, columnIndices, scale, shape, inverseLength, fixedInverseLength, fixedScale, fixedShape);
+      return cov;
+    }
+
+    private static IList<double> GetGradient(double[,] x, int i, int j, int[] columnIndices, double scale, double shape, double inverseLength,
+      bool fixedInverseLength, bool fixedScale, bool fixedShape) {
+      double d = i == j
+                   ? 0.0
+                   : Util.SqrDist(x, i, j, columnIndices, inverseLength);
+
+      double b = 1 + 0.5 * d / shape;
+      var g = new List<double>(3);
+      if (!fixedInverseLength) g.Add(scale * Math.Pow(b, -shape - 1) * d);
+      if (!fixedScale) g.Add(2 * scale * Math.Pow(b, -shape));
+      if (!fixedShape) g.Add(scale * Math.Pow(b, -shape) * (0.5 * d / b - shape * Math.Log(b)));
+      return g;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceScale.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceScale.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceScale.cs	(revision 17041)
@@ -0,0 +1,112 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("4871900E-8B7A-4D74-969A-773D63198733")]
+  [Item(Name = "CovarianceScale",
+    Description = "Scale covariance function for Gaussian processes.")]
+  public sealed class CovarianceScale : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+    private bool HasFixedScaleParameter {
+      get { return ScaleParameter.Value != null; }
+    }
+
+    public IValueParameter<ICovarianceFunction> CovarianceFunctionParameter {
+      get { return (IValueParameter<ICovarianceFunction>)Parameters["CovarianceFunction"]; }
+    }
+
+    [StorableConstructor]
+    private CovarianceScale(StorableConstructorFlag _) : base(_) {
+    }
+
+    private CovarianceScale(CovarianceScale original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovarianceScale()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale parameter."));
+      Parameters.Add(new ValueParameter<ICovarianceFunction>("CovarianceFunction", "The covariance function that should be scaled.", new CovarianceSquaredExponentialIso()));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceScale(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return (HasFixedScaleParameter ? 0 : 1) + CovarianceFunctionParameter.Value.GetNumberOfParameters(numberOfVariables);
+    }
+
+    public void SetParameter(double[] p) {
+      double scale;
+      GetParameterValues(p, out scale);
+      ScaleParameter.Value = new DoubleValue(scale);
+      CovarianceFunctionParameter.Value.SetParameter(p.Skip(1).ToArray());
+    }
+
+    private void GetParameterValues(double[] p, out double scale) {
+      // gather parameter values
+      if (HasFixedScaleParameter) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[0]);
+      }
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      double scale;
+      GetParameterValues(p, out scale);
+      var fixedScale = HasFixedScaleParameter;
+      var subCov = CovarianceFunctionParameter.Value.GetParameterizedCovarianceFunction(p.Skip(1).ToArray(), columnIndices);
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => scale * subCov.Covariance(x, i, j);
+      cov.CrossCovariance = (x, xt, i, j) => scale * subCov.CrossCovariance(x, xt, i, j);
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, columnIndices, scale, subCov, fixedScale);
+      return cov;
+    }
+
+    private static IList<double> GetGradient(double[,] x, int i, int j, int[] columnIndices, double scale, ParameterizedCovarianceFunction cov,
+      bool fixedScale) {
+      var gr = new List<double>((!fixedScale ? 1 : 0) + cov.CovarianceGradient(x, i, j).Count);
+      if (!fixedScale) {
+        gr.Add(2 * scale * cov.Covariance(x, i, j));
+      }
+      foreach (var g in cov.CovarianceGradient(x, i, j))
+        gr.Add(scale * g);
+      return gr;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSpectralMixture.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSpectralMixture.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSpectralMixture.cs	(revision 17041)
@@ -0,0 +1,239 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("FF180FAF-851D-425E-AC0B-25930CE9604C")]
+  [Item(Name = "CovarianceSpectralMixture",
+    Description = "The spectral mixture kernel described in Wilson A. G. and Adams R.P., Gaussian Process Kernels for Pattern Discovery and Exptrapolation, ICML 2013.")]
+  public sealed class CovarianceSpectralMixture : ParameterizedNamedItem, ICovarianceFunction {
+    public const string QParameterName = "Number of components (Q)";
+    public const string WeightParameterName = "Weight";
+    public const string FrequencyParameterName = "Component frequency (mu)";
+    public const string LengthScaleParameterName = "Length scale (nu)";
+    public IValueParameter<IntValue> QParameter {
+      get { return (IValueParameter<IntValue>)Parameters[QParameterName]; }
+    }
+
+    public IValueParameter<DoubleArray> WeightParameter {
+      get { return (IValueParameter<DoubleArray>)Parameters[WeightParameterName]; }
+    }
+    public IValueParameter<DoubleArray> FrequencyParameter {
+      get { return (IValueParameter<DoubleArray>)Parameters[FrequencyParameterName]; }
+    }
+
+    public IValueParameter<DoubleArray> LengthScaleParameter {
+      get { return (IValueParameter<DoubleArray>)Parameters[LengthScaleParameterName]; }
+    }
+
+    private bool HasFixedWeightParameter {
+      get { return WeightParameter.Value != null; }
+    }
+    private bool HasFixedFrequencyParameter {
+      get { return FrequencyParameter.Value != null; }
+    }
+    private bool HasFixedLengthScaleParameter {
+      get { return LengthScaleParameter.Value != null; }
+    }
+
+    [StorableConstructor]
+    private CovarianceSpectralMixture(StorableConstructorFlag _) : base(_) {
+    }
+
+    private CovarianceSpectralMixture(CovarianceSpectralMixture original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovarianceSpectralMixture()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+      Parameters.Add(new ValueParameter<IntValue>(QParameterName, "The number of Gaussians (Q) to use for the spectral mixture.", new IntValue(10)));
+      Parameters.Add(new OptionalValueParameter<DoubleArray>(WeightParameterName, "The weight of the component w (peak height of the Gaussian in spectrum)."));
+      Parameters.Add(new OptionalValueParameter<DoubleArray>(FrequencyParameterName, "The inverse component period parameter mu_q (location of the Gaussian in spectrum)."));
+      Parameters.Add(new OptionalValueParameter<DoubleArray>(LengthScaleParameterName, "The length scale parameter (nu_q) (variance of the Gaussian in the spectrum)."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceSpectralMixture(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      var q = QParameter.Value.Value;
+      return
+        (HasFixedWeightParameter ? 0 : q) +
+        (HasFixedFrequencyParameter ? 0 : q * numberOfVariables) +
+        (HasFixedLengthScaleParameter ? 0 : q * numberOfVariables);
+    }
+
+    public void SetParameter(double[] p) {
+      double[] weight, frequency, lengthScale;
+      GetParameterValues(p, out weight, out frequency, out lengthScale);
+      WeightParameter.Value = new DoubleArray(weight);
+      FrequencyParameter.Value = new DoubleArray(frequency);
+      LengthScaleParameter.Value = new DoubleArray(lengthScale);
+    }
+
+
+    private void GetParameterValues(double[] p, out double[] weight, out double[] frequency, out double[] lengthScale) {
+      // gather parameter values
+      int c = 0;
+      int q = QParameter.Value.Value;
+      // guess number of elements for frequency and length (=q * numberOfVariables)
+      int n = WeightParameter.Value == null ? ((p.Length - q) / 2) : (p.Length / 2);
+      if (HasFixedWeightParameter) {
+        weight = WeightParameter.Value.ToArray();
+      } else {
+        weight = p.Skip(c).Select(Math.Exp).Take(q).ToArray();
+        c += q;
+      }
+      if (HasFixedFrequencyParameter) {
+        frequency = FrequencyParameter.Value.ToArray();
+      } else {
+        frequency = p.Skip(c).Select(Math.Exp).Take(n).ToArray();
+        c += n;
+      }
+      if (HasFixedLengthScaleParameter) {
+        lengthScale = LengthScaleParameter.Value.ToArray();
+      } else {
+        lengthScale = p.Skip(c).Select(Math.Exp).Take(n).ToArray();
+        c += n;
+      }
+      if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceSpectralMixture", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      double[] weight, frequency, lengthScale;
+      GetParameterValues(p, out weight, out frequency, out lengthScale);
+      var fixedWeight = HasFixedWeightParameter;
+      var fixedFrequency = HasFixedFrequencyParameter;
+      var fixedLengthScale = HasFixedLengthScaleParameter;
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => {
+        return GetCovariance(x, x, i, j, QParameter.Value.Value, weight, frequency,
+                             lengthScale, columnIndices);
+      };
+      cov.CrossCovariance = (x, xt, i, j) => {
+        return GetCovariance(x, xt, i, j, QParameter.Value.Value, weight, frequency,
+                             lengthScale, columnIndices);
+      };
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, QParameter.Value.Value, weight, frequency,
+                             lengthScale, columnIndices, fixedWeight, fixedFrequency, fixedLengthScale);
+      return cov;
+    }
+
+    private static double GetCovariance(double[,] x, double[,] xt, int i, int j, int maxQ, double[] weight, double[] frequency, double[] lengthScale, int[] columnIndices) {
+      // tau = x - x' (only for selected variables)
+      double[] tau =
+        Util.GetRow(x, i, columnIndices).Zip(Util.GetRow(xt, j, columnIndices), (xi, xj) => xi - xj).ToArray();
+      int numberOfVariables = lengthScale.Length / maxQ;
+      double k = 0;
+      // for each component
+      for (int q = 0; q < maxQ; q++) {
+        double kc = weight[q]; // weighted kernel component
+
+        int idx = 0; // helper index for tau
+        // for each selected variable
+        for (int c = 0; c < columnIndices.Length; c++) {
+          var col = columnIndices[c];
+          kc *= f1(tau[idx], lengthScale[q * numberOfVariables + col]) * f2(tau[idx], frequency[q * numberOfVariables + col]);
+          idx++;
+        }
+        k += kc;
+      }
+      return k;
+    }
+
+    public static double f1(double tau, double lengthScale) {
+      return Math.Exp(-2 * Math.PI * Math.PI * tau * tau * lengthScale);
+    }
+    public static double f2(double tau, double frequency) {
+      return Math.Cos(2 * Math.PI * tau * frequency);
+    }
+
+    // order of returned gradients must match the order in GetParameterValues!
+    private static IList<double> GetGradient(double[,] x, int i, int j, int maxQ, double[] weight, double[] frequency, double[] lengthScale, int[] columnIndices,
+      bool fixedWeight, bool fixedFrequency, bool fixedLengthScale) {
+      double[] tau = Util.GetRow(x, i, columnIndices).Zip(Util.GetRow(x, j, columnIndices), (xi, xj) => xi - xj).ToArray();
+      int numberOfVariables = lengthScale.Length / maxQ;
+
+      var g = new List<double>((!fixedWeight ? maxQ : 0) + (!fixedFrequency ? maxQ * columnIndices.Length : 0) + (!fixedLengthScale ? maxQ * columnIndices.Length : 0));
+      if (!fixedWeight) {
+        // weight
+        // for each component
+        for (int q = 0; q < maxQ; q++) {
+          double k = weight[q];
+          int idx = 0; // helper index for tau
+          // for each selected variable
+          for (int c = 0; c < columnIndices.Length; c++) {
+            var col = columnIndices[c];
+            k *= f1(tau[idx], lengthScale[q * numberOfVariables + col]) * f2(tau[idx], frequency[q * numberOfVariables + col]);
+            idx++;
+          }
+          g.Add(k);
+        }
+      }
+
+      if (!fixedFrequency) {
+        // frequency
+        // for each component
+        for (int q = 0; q < maxQ; q++) {
+          int idx = 0; // helper index for tau
+          // for each selected variable
+          foreach (var c in columnIndices) {
+            double k = f1(tau[idx], lengthScale[q * numberOfVariables + c]) *
+                       -2 * Math.PI * tau[idx] * frequency[q * numberOfVariables + c] *
+                       Math.Sin(2 * Math.PI * tau[idx] * frequency[q * numberOfVariables + c]);
+            idx++;
+            g.Add(weight[q] * k);
+          }
+        }
+      }
+
+      if (!fixedLengthScale) {
+        // length scale
+        // for each component
+        for (int q = 0; q < maxQ; q++) {
+          int idx = 0; // helper index for tau
+          // for each selected variable
+          foreach (var c in columnIndices) {
+            double k = -2 * Math.PI * Math.PI * tau[idx] * tau[idx] * lengthScale[q * numberOfVariables + c] *
+                       f1(tau[idx], lengthScale[q * numberOfVariables + c]) *
+                       f2(tau[idx], frequency[q * numberOfVariables + c]);
+            idx++;
+            g.Add(weight[q] * k);
+          }
+        }
+      }
+
+      return g;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialArd.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialArd.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialArd.cs	(revision 17041)
@@ -0,0 +1,144 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("88CB6895-6CD8-4F81-8E3D-905EEBB7E77A")]
+  [Item(Name = "CovarianceSquaredExponentialArd", Description = "Squared exponential covariance function with automatic relevance determination for Gaussian processes.")]
+  public sealed class CovarianceSquaredExponentialArd : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+
+    public IValueParameter<DoubleArray> InverseLengthParameter {
+      get { return (IValueParameter<DoubleArray>)Parameters["InverseLength"]; }
+    }
+    private bool HasFixedInverseLengthParameter {
+      get { return InverseLengthParameter.Value != null; }
+    }
+    private bool HasFixedScaleParameter {
+      get { return ScaleParameter.Value != null; }
+    }
+
+    [StorableConstructor]
+    private CovarianceSquaredExponentialArd(StorableConstructorFlag _) : base(_) { }
+    private CovarianceSquaredExponentialArd(CovarianceSquaredExponentialArd original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public CovarianceSquaredExponentialArd()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale parameter of the squared exponential covariance function with ARD."));
+      Parameters.Add(new OptionalValueParameter<DoubleArray>("InverseLength", "The inverse length parameter for automatic relevance determination."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceSquaredExponentialArd(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return
+        (HasFixedScaleParameter ? 0 : 1) +
+        (HasFixedInverseLengthParameter ? 0 : numberOfVariables);
+    }
+
+    public void SetParameter(double[] p) {
+      double scale;
+      double[] inverseLength;
+      GetParameterValues(p, out scale, out inverseLength);
+      ScaleParameter.Value = new DoubleValue(scale);
+      InverseLengthParameter.Value = new DoubleArray(inverseLength);
+    }
+
+    private void GetParameterValues(double[] p, out double scale, out double[] inverseLength) {
+      int c = 0;
+      // gather parameter values
+      if (HasFixedInverseLengthParameter) {
+        inverseLength = InverseLengthParameter.Value.ToArray();
+      } else {
+        int length = p.Length;
+        if (!HasFixedScaleParameter) length--;
+        inverseLength = p.Select(e => 1.0 / Math.Exp(e)).Take(length).ToArray();
+        c += inverseLength.Length;
+      }
+      if (HasFixedScaleParameter) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[c]);
+        c++;
+      }
+      if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceSquaredExponentialArd", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      double scale;
+      double[] inverseLength;
+      GetParameterValues(p, out scale, out inverseLength);
+      var fixedInverseLength = HasFixedInverseLengthParameter;
+      var fixedScale = HasFixedScaleParameter;
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => {
+        double d = i == j
+                 ? 0.0
+                 : Util.SqrDist(x, i, j, inverseLength, columnIndices);
+        return scale * Math.Exp(-d / 2.0);
+      };
+      cov.CrossCovariance = (x, xt, i, j) => {
+        double d = Util.SqrDist(x, i, xt, j, inverseLength, columnIndices);
+        return scale * Math.Exp(-d / 2.0);
+      };
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, columnIndices, scale, inverseLength, fixedInverseLength, fixedScale);
+      return cov;
+    }
+
+    // order of returned gradients must match the order in GetParameterValues!
+    private static IList<double> GetGradient(double[,] x, int i, int j, int[] columnIndices, double scale, double[] inverseLength,
+      bool fixedInverseLength, bool fixedScale) {
+      double d = i == j
+                   ? 0.0
+                   : Util.SqrDist(x, i, j, inverseLength, columnIndices);
+
+      int k = 0;
+      var g = new List<double>((!fixedInverseLength ? columnIndices.Length : 0) + (!fixedScale ? 1 : 0));
+      if (!fixedInverseLength) {
+        for (int c = 0; c < columnIndices.Length; c++) {
+          var columnIndex = columnIndices[c];
+          double sqrDist = Util.SqrDist(x[i, columnIndex] * inverseLength[k], x[j, columnIndex] * inverseLength[k]);
+          g.Add(scale * Math.Exp(-d / 2.0) * sqrDist);
+          k++;
+        }
+      }
+      if (!fixedScale) g.Add(2.0 * scale * Math.Exp(-d / 2.0));
+      return g;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialIso.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialIso.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialIso.cs	(revision 17041)
@@ -0,0 +1,139 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("1335C8EF-73CA-40F4-9124-EC6D7E3C68E0")]
+  [Item(Name = "CovarianceSquaredExponentialIso",
+    Description = "Isotropic squared exponential covariance function for Gaussian processes.")]
+  public sealed class CovarianceSquaredExponentialIso : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+
+    public IValueParameter<DoubleValue> InverseLengthParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["InverseLength"]; }
+    }
+
+    private bool HasFixedInverseLengthParameter {
+      get { return InverseLengthParameter.Value != null; }
+    }
+    private bool HasFixedScaleParameter {
+      get { return ScaleParameter.Value != null; }
+    }
+
+    [StorableConstructor]
+    private CovarianceSquaredExponentialIso(StorableConstructorFlag _) : base(_) {
+    }
+
+    private CovarianceSquaredExponentialIso(CovarianceSquaredExponentialIso original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovarianceSquaredExponentialIso()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale parameter of the isometric squared exponential covariance function."));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("InverseLength", "The inverse length parameter of the isometric squared exponential covariance function."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceSquaredExponentialIso(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return
+        (HasFixedScaleParameter ? 0 : 1) +
+        (HasFixedInverseLengthParameter ? 0 : 1);
+    }
+
+    public void SetParameter(double[] p) {
+      double scale, inverseLength;
+      GetParameterValues(p, out scale, out inverseLength);
+      ScaleParameter.Value = new DoubleValue(scale);
+      InverseLengthParameter.Value = new DoubleValue(inverseLength);
+    }
+
+
+    private void GetParameterValues(double[] p, out double scale, out double inverseLength) {
+      // gather parameter values
+      int c = 0;
+      if (HasFixedInverseLengthParameter) {
+        inverseLength = InverseLengthParameter.Value.Value;
+      } else {
+        inverseLength = 1.0 / Math.Exp(p[c]);
+        c++;
+      }
+
+      if (HasFixedScaleParameter) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[c]);
+        c++;
+      }
+      if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceSquaredExponentialIso", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      double inverseLength, scale;
+      GetParameterValues(p, out scale, out inverseLength);
+      var fixedInverseLength = HasFixedInverseLengthParameter;
+      var fixedScale = HasFixedScaleParameter;
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => {
+        double d = i == j
+                ? 0.0
+                : Util.SqrDist(x, i, j, columnIndices, inverseLength);
+        return scale * Math.Exp(-d / 2.0);
+      };
+      cov.CrossCovariance = (x, xt, i, j) => {
+        double d = Util.SqrDist(x, i, xt, j, columnIndices, inverseLength);
+        return scale * Math.Exp(-d / 2.0);
+      };
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, scale, inverseLength, columnIndices,
+        fixedInverseLength, fixedScale);
+      return cov;
+    }
+
+    // order of returned gradients must match the order in GetParameterValues!
+    private static IList<double> GetGradient(double[,] x, int i, int j, double sf2, double inverseLength, int[] columnIndices,
+      bool fixedInverseLength, bool fixedScale) {
+      double d = i == j
+                   ? 0.0
+                   : Util.SqrDist(x, i, j, columnIndices, inverseLength);
+      double g = Math.Exp(-d / 2.0);
+      var gr = new List<double>(2);
+      if (!fixedInverseLength) gr.Add(sf2 * g * d);
+      if (!fixedScale) gr.Add(2.0 * sf2 * g);
+      return gr;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSum.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSum.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSum.cs	(revision 17041)
@@ -0,0 +1,97 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("8F1A684A-98BE-429A-BDA2-E1FB7DDF09F0")]
+  [Item(Name = "CovarianceSum",
+    Description = "Sum covariance function for Gaussian processes.")]
+  public sealed class CovarianceSum : Item, ICovarianceFunction {
+    [Storable]
+    private ItemList<ICovarianceFunction> terms;
+
+    [Storable]
+    private int numberOfVariables;
+    public ItemList<ICovarianceFunction> Terms {
+      get { return terms; }
+    }
+
+    [StorableConstructor]
+    private CovarianceSum(StorableConstructorFlag _) : base(_) {
+    }
+
+    private CovarianceSum(CovarianceSum original, Cloner cloner)
+      : base(original, cloner) {
+      this.terms = cloner.Clone(original.terms);
+      this.numberOfVariables = original.numberOfVariables;
+    }
+
+    public CovarianceSum()
+      : base() {
+      this.terms = new ItemList<ICovarianceFunction>();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceSum(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      this.numberOfVariables = numberOfVariables;
+      return terms.Select(t => t.GetNumberOfParameters(numberOfVariables)).Sum();
+    }
+
+    public void SetParameter(double[] p) {
+      int offset = 0;
+      foreach (var t in terms) {
+        var numberOfParameters = t.GetNumberOfParameters(numberOfVariables);
+        t.SetParameter(p.Skip(offset).Take(numberOfParameters).ToArray());
+        offset += numberOfParameters;
+      }
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      if (terms.Count == 0) throw new ArgumentException("at least one term is necessary for the product covariance function.");
+      var functions = new List<ParameterizedCovarianceFunction>();
+      foreach (var t in terms) {
+        var numberOfParameters = t.GetNumberOfParameters(numberOfVariables);
+        functions.Add(t.GetParameterizedCovarianceFunction(p.Take(numberOfParameters).ToArray(), columnIndices));
+        p = p.Skip(numberOfParameters).ToArray();
+      }
+
+      var sum = new ParameterizedCovarianceFunction();
+      sum.Covariance = (x, i, j) => functions.Select(e => e.Covariance(x, i, j)).Sum();
+      sum.CrossCovariance = (x, xt, i, j) => functions.Select(e => e.CrossCovariance(x, xt, i, j)).Sum();
+      sum.CovarianceGradient = (x, i, j) => {
+        var g = new List<double>();
+        foreach (var e in functions)
+          g.AddRange(e.CovarianceGradient(x, i, j));
+        return g;
+      };
+      return sum;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessBase.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessBase.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessBase.cs	(revision 17041)
@@ -0,0 +1,219 @@
+
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Linq;
+using HeuristicLab.Algorithms.GradientDescent;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Operators;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Base class for Gaussian process data analysis algorithms (regression and classification).
+  /// </summary>
+  [StorableType("A5070F15-8E44-44DC-92E1-000826E933D3")]
+  public abstract class GaussianProcessBase : EngineAlgorithm {
+    protected const string MeanFunctionParameterName = "MeanFunction";
+    protected const string CovarianceFunctionParameterName = "CovarianceFunction";
+    protected const string MinimizationIterationsParameterName = "Iterations";
+    protected const string ApproximateGradientsParameterName = "ApproximateGradients";
+    protected const string SeedParameterName = "Seed";
+    protected const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+    protected const string ModelCreatorParameterName = "GaussianProcessModelCreator";
+    protected const string NegativeLogLikelihoodParameterName = "NegativeLogLikelihood";
+    protected const string HyperparameterParameterName = "Hyperparameter";
+    protected const string HyperparameterGradientsParameterName = "HyperparameterGradients";
+    protected const string SolutionCreatorParameterName = "GaussianProcessSolutionCreator";
+    protected const string ScaleInputValuesParameterName = "ScaleInputValues";
+
+    public new IDataAnalysisProblem Problem {
+      get { return (IDataAnalysisProblem)base.Problem; }
+      set { base.Problem = value; }
+    }
+
+    #region parameter properties
+    public IValueParameter<IMeanFunction> MeanFunctionParameter {
+      get { return (IValueParameter<IMeanFunction>)Parameters[MeanFunctionParameterName]; }
+    }
+    public IValueParameter<ICovarianceFunction> CovarianceFunctionParameter {
+      get { return (IValueParameter<ICovarianceFunction>)Parameters[CovarianceFunctionParameterName]; }
+    }
+    public IValueParameter<IntValue> MinimizationIterationsParameter {
+      get { return (IValueParameter<IntValue>)Parameters[MinimizationIterationsParameterName]; }
+    }
+    public IValueParameter<IntValue> SeedParameter {
+      get { return (IValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+    public IValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (IValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> ScaleInputValuesParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[ScaleInputValuesParameterName]; }
+    }
+    #endregion
+    #region properties
+    public IMeanFunction MeanFunction {
+      set { MeanFunctionParameter.Value = value; }
+      get { return MeanFunctionParameter.Value; }
+    }
+    public ICovarianceFunction CovarianceFunction {
+      set { CovarianceFunctionParameter.Value = value; }
+      get { return CovarianceFunctionParameter.Value; }
+    }
+    public int MinimizationIterations {
+      set { MinimizationIterationsParameter.Value.Value = value; }
+      get { return MinimizationIterationsParameter.Value.Value; }
+    }
+    public int Seed { get { return SeedParameter.Value.Value; } set { SeedParameter.Value.Value = value; } }
+    public bool SetSeedRandomly { get { return SetSeedRandomlyParameter.Value.Value; } set { SetSeedRandomlyParameter.Value.Value = value; } }
+
+    public bool ScaleInputValues {
+      get { return ScaleInputValuesParameter.Value.Value; }
+      set { ScaleInputValuesParameter.Value.Value = value; }
+    }
+    #endregion
+
+    [StorableConstructor]
+    protected GaussianProcessBase(StorableConstructorFlag _) : base(_) { }
+    protected GaussianProcessBase(GaussianProcessBase original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    protected GaussianProcessBase(IDataAnalysisProblem problem)
+      : base() {
+      Problem = problem;
+      Parameters.Add(new ValueParameter<IMeanFunction>(MeanFunctionParameterName, "The mean function to use.", new MeanConst()));
+      Parameters.Add(new ValueParameter<ICovarianceFunction>(CovarianceFunctionParameterName, "The covariance function to use.", new CovarianceSquaredExponentialIso()));
+      Parameters.Add(new ValueParameter<IntValue>(MinimizationIterationsParameterName, "The number of iterations for likelihood optimization with LM-BFGS.", new IntValue(20)));
+      Parameters.Add(new ValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+      Parameters.Add(new ValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+
+      Parameters.Add(new ValueParameter<BoolValue>(ApproximateGradientsParameterName, "Indicates that gradients should not be approximated (necessary for LM-BFGS).", new BoolValue(false)));
+      Parameters[ApproximateGradientsParameterName].Hidden = true; // should not be changed
+
+      Parameters.Add(new FixedValueParameter<BoolValue>(ScaleInputValuesParameterName,
+        "Determines if the input variable values are scaled to the range [0..1] for training.", new BoolValue(true)));
+      Parameters[ScaleInputValuesParameterName].Hidden = true;
+
+      // necessary for BFGS
+      Parameters.Add(new FixedValueParameter<BoolValue>("Maximization (BFGS)", new BoolValue(false)));
+      Parameters["Maximization (BFGS)"].Hidden = true;
+
+      var randomCreator = new HeuristicLab.Random.RandomCreator();
+      var gpInitializer = new GaussianProcessHyperparameterInitializer();
+      var bfgsInitializer = new LbfgsInitializer();
+      var makeStep = new LbfgsMakeStep();
+      var branch = new ConditionalBranch();
+      var modelCreator = new Placeholder();
+      var updateResults = new LbfgsUpdateResults();
+      var analyzer = new LbfgsAnalyzer();
+      var finalModelCreator = new Placeholder();
+      var finalAnalyzer = new LbfgsAnalyzer();
+      var solutionCreator = new Placeholder();
+
+      OperatorGraph.InitialOperator = randomCreator;
+      randomCreator.SeedParameter.ActualName = SeedParameterName;
+      randomCreator.SeedParameter.Value = null;
+      randomCreator.SetSeedRandomlyParameter.ActualName = SetSeedRandomlyParameterName;
+      randomCreator.SetSeedRandomlyParameter.Value = null;
+      randomCreator.Successor = gpInitializer;
+
+      gpInitializer.CovarianceFunctionParameter.ActualName = CovarianceFunctionParameterName;
+      gpInitializer.MeanFunctionParameter.ActualName = MeanFunctionParameterName;
+      gpInitializer.ProblemDataParameter.ActualName = Problem.ProblemDataParameter.Name;
+      gpInitializer.HyperparameterParameter.ActualName = HyperparameterParameterName;
+      gpInitializer.RandomParameter.ActualName = randomCreator.RandomParameter.Name;
+      gpInitializer.Successor = bfgsInitializer;
+
+      bfgsInitializer.IterationsParameter.ActualName = MinimizationIterationsParameterName;
+      bfgsInitializer.PointParameter.ActualName = HyperparameterParameterName;
+      bfgsInitializer.ApproximateGradientsParameter.ActualName = ApproximateGradientsParameterName;
+      bfgsInitializer.Successor = makeStep;
+
+      makeStep.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
+      makeStep.PointParameter.ActualName = HyperparameterParameterName;
+      makeStep.Successor = branch;
+
+      branch.ConditionParameter.ActualName = makeStep.TerminationCriterionParameter.Name;
+      branch.FalseBranch = modelCreator;
+      branch.TrueBranch = finalModelCreator;
+
+      modelCreator.OperatorParameter.ActualName = ModelCreatorParameterName;
+      modelCreator.Successor = updateResults;
+
+      updateResults.MaximizationParameter.ActualName = "Maximization (BFGS)";
+      updateResults.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
+      updateResults.QualityParameter.ActualName = NegativeLogLikelihoodParameterName;
+      updateResults.QualityGradientsParameter.ActualName = HyperparameterGradientsParameterName;
+      updateResults.ApproximateGradientsParameter.ActualName = ApproximateGradientsParameterName;
+      updateResults.Successor = analyzer;
+
+      analyzer.QualityParameter.ActualName = NegativeLogLikelihoodParameterName;
+      analyzer.PointParameter.ActualName = HyperparameterParameterName;
+      analyzer.QualityGradientsParameter.ActualName = HyperparameterGradientsParameterName;
+      analyzer.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
+      analyzer.PointsTableParameter.ActualName = "Hyperparameter table";
+      analyzer.QualityGradientsTableParameter.ActualName = "Gradients table";
+      analyzer.QualitiesTableParameter.ActualName = "Negative log likelihood table";
+      analyzer.Successor = makeStep;
+
+      finalModelCreator.OperatorParameter.ActualName = ModelCreatorParameterName;
+      finalModelCreator.Successor = finalAnalyzer;
+
+      finalAnalyzer.QualityParameter.ActualName = NegativeLogLikelihoodParameterName;
+      finalAnalyzer.PointParameter.ActualName = HyperparameterParameterName;
+      finalAnalyzer.QualityGradientsParameter.ActualName = HyperparameterGradientsParameterName;
+      finalAnalyzer.PointsTableParameter.ActualName = analyzer.PointsTableParameter.ActualName;
+      finalAnalyzer.QualityGradientsTableParameter.ActualName = analyzer.QualityGradientsTableParameter.ActualName;
+      finalAnalyzer.QualitiesTableParameter.ActualName = analyzer.QualitiesTableParameter.ActualName;
+      finalAnalyzer.Successor = solutionCreator;
+
+      solutionCreator.OperatorParameter.ActualName = SolutionCreatorParameterName;
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.4
+      #region Backwards compatible code, remove with 3.5
+      if (Parameters.ContainsKey("Maximization")) {
+        Parameters.Remove("Maximization");
+      }
+
+      if (!Parameters.ContainsKey("Maximization (BFGS)")) {
+        Parameters.Add(new FixedValueParameter<BoolValue>("Maximization (BFGS)", new BoolValue(false)));
+        Parameters["Maximization (BFGS)"].Hidden = true;
+        OperatorGraph.Operators.OfType<LbfgsUpdateResults>().First().MaximizationParameter.ActualName = "Maximization BFGS";
+      }
+
+      if (!Parameters.ContainsKey(ScaleInputValuesParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(ScaleInputValuesParameterName,
+          "Determines if the input variable values are scaled to the range [0..1] for training.", new BoolValue(true)));
+        Parameters[ScaleInputValuesParameterName].Hidden = true;
+      }
+      #endregion
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassification.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassification.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassification.cs	(revision 17041)
@@ -0,0 +1,153 @@
+
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.PluginInfrastructure;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Gaussian process least-squares classification data analysis algorithm.
+  /// </summary>
+  [Item("Gaussian Process Least-Squares Classification", "Gaussian process least-squares classification data analysis algorithm.")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 160)]
+  [StorableType("5D8711E4-1A3F-45E7-83A5-E9BBAC239793")]
+  public sealed class GaussianProcessClassification : GaussianProcessBase, IStorableContent {
+    public string Filename { get; set; }
+
+    public override Type ProblemType { get { return typeof(IClassificationProblem); } }
+    public new IClassificationProblem Problem {
+      get { return (IClassificationProblem)base.Problem; }
+      set { base.Problem = value; }
+    }
+
+    private const string ModelParameterName = "Model";
+    private const string CreateSolutionParameterName = "CreateSolution";
+
+    #region parameter properties
+    public IConstrainedValueParameter<IGaussianProcessClassificationModelCreator> GaussianProcessModelCreatorParameter {
+      get { return (IConstrainedValueParameter<IGaussianProcessClassificationModelCreator>)Parameters[ModelCreatorParameterName]; }
+    }
+    public IFixedValueParameter<GaussianProcessClassificationSolutionCreator> GaussianProcessSolutionCreatorParameter {
+      get { return (IFixedValueParameter<GaussianProcessClassificationSolutionCreator>)Parameters[SolutionCreatorParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> CreateSolutionParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
+    }
+    #endregion
+    #region properties
+    public bool CreateSolution {
+      get { return CreateSolutionParameter.Value.Value; }
+      set { CreateSolutionParameter.Value.Value = value; }
+    }
+    #endregion
+
+    [StorableConstructor]
+    private GaussianProcessClassification(StorableConstructorFlag _) : base(_) { }
+    private GaussianProcessClassification(GaussianProcessClassification original, Cloner cloner)
+      : base(original, cloner) {
+      RegisterEventHandlers();
+    }
+    public GaussianProcessClassification()
+      : base(new ClassificationProblem()) {
+      this.name = ItemName;
+      this.description = ItemDescription;
+
+      var modelCreators = ApplicationManager.Manager.GetInstances<IGaussianProcessClassificationModelCreator>();
+      var defaultModelCreator = modelCreators.First(c => c is GaussianProcessClassificationModelCreator);
+
+      // GP regression and classification algorithms only differ in the model and solution creators,
+      // thus we use a common base class and use operator parameters to implement the specific versions.
+      // Different model creators can be implemented,
+      // but the solution creator is implemented in a generic fashion already and we don't allow derived solution creators
+      Parameters.Add(new ConstrainedValueParameter<IGaussianProcessClassificationModelCreator>(ModelCreatorParameterName, "The operator to create the Gaussian process model.",
+        new ItemSet<IGaussianProcessClassificationModelCreator>(modelCreators), defaultModelCreator));
+      // this parameter is not intended to be changed, 
+      Parameters.Add(new FixedValueParameter<GaussianProcessClassificationSolutionCreator>(SolutionCreatorParameterName, "The solution creator for the algorithm",
+        new GaussianProcessClassificationSolutionCreator()));
+      Parameters[SolutionCreatorParameterName].Hidden = true;
+      // TODO: it would be better to deactivate the solution creator when this parameter is changed
+      Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+      Parameters[CreateSolutionParameterName].Hidden = true;
+
+      ParameterizedModelCreators();
+      ParameterizeSolutionCreator(GaussianProcessSolutionCreatorParameter.Value);
+      RegisterEventHandlers();
+    }
+
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
+      if (!Parameters.ContainsKey(CreateSolutionParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+        Parameters[CreateSolutionParameterName].Hidden = true;
+      }
+      #endregion
+      RegisterEventHandlers();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessClassification(this, cloner);
+    }
+
+    #region events
+    private void RegisterEventHandlers() {
+      GaussianProcessModelCreatorParameter.ValueChanged += ModelCreatorParameter_ValueChanged;
+    }
+
+    private void ModelCreatorParameter_ValueChanged(object sender, EventArgs e) {
+      ParameterizedModelCreator(GaussianProcessModelCreatorParameter.Value);
+    }
+    #endregion
+
+    private void ParameterizedModelCreators() {
+      foreach (var creator in GaussianProcessModelCreatorParameter.ValidValues) {
+        ParameterizedModelCreator(creator);
+      }
+    }
+
+    private void ParameterizedModelCreator(IGaussianProcessClassificationModelCreator modelCreator) {
+      modelCreator.ProblemDataParameter.ActualName = Problem.ProblemDataParameter.Name;
+      modelCreator.MeanFunctionParameter.ActualName = MeanFunctionParameterName;
+      modelCreator.CovarianceFunctionParameter.ActualName = CovarianceFunctionParameterName;
+
+      // parameter names fixed by the algorithm
+      modelCreator.ModelParameter.ActualName = ModelParameterName;
+      modelCreator.HyperparameterParameter.ActualName = HyperparameterParameterName;
+      modelCreator.HyperparameterGradientsParameter.ActualName = HyperparameterGradientsParameterName;
+      modelCreator.NegativeLogLikelihoodParameter.ActualName = NegativeLogLikelihoodParameterName;
+    }
+
+    private void ParameterizeSolutionCreator(GaussianProcessClassificationSolutionCreator solutionCreator) {
+      solutionCreator.ModelParameter.ActualName = ModelParameterName;
+      solutionCreator.ProblemDataParameter.ActualName = Problem.ProblemDataParameter.Name;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationModelCreator.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationModelCreator.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationModelCreator.cs	(revision 17041)
@@ -0,0 +1,81 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("C8A3E25F-C9DF-4BBB-B824-27A564A90E39")]
+  [Item(Name = "GaussianProcessClassificationModelCreator",
+    Description = "Creates a Gaussian process model for least-squares classification given the data, the hyperparameters, a mean function, and a covariance function.")]
+  public sealed class GaussianProcessClassificationModelCreator : GaussianProcessModelCreator, IGaussianProcessClassificationModelCreator {
+    private const string ProblemDataParameterName = "ProblemData";
+
+    #region Parameter Properties
+    public ILookupParameter<IClassificationProblemData> ProblemDataParameter {
+      get { return (ILookupParameter<IClassificationProblemData>)Parameters[ProblemDataParameterName]; }
+    }
+    #endregion
+
+    #region Properties
+    private IClassificationProblemData ProblemData {
+      get { return ProblemDataParameter.ActualValue; }
+    }
+    #endregion
+    [StorableConstructor]
+    private GaussianProcessClassificationModelCreator(StorableConstructorFlag _) : base(_) { }
+    private GaussianProcessClassificationModelCreator(GaussianProcessClassificationModelCreator original, Cloner cloner) : base(original, cloner) { }
+    public GaussianProcessClassificationModelCreator()
+      : base() {
+      Parameters.Add(new LookupParameter<IClassificationProblemData>(ProblemDataParameterName, "The classification problem data for the Gaussian process model."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessClassificationModelCreator(this, cloner);
+    }
+
+    public override IOperation Apply() {
+      try {
+        var model = Create(ProblemData, Hyperparameter.ToArray(), MeanFunction, CovarianceFunction, ScaleInputValues);
+        ModelParameter.ActualValue = model;
+        NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(model.NegativeLogLikelihood);
+        HyperparameterGradientsParameter.ActualValue = new RealVector(model.HyperparameterGradients);
+        return base.Apply();
+      } catch (ArgumentException) {
+      } catch (alglib.alglibexception) {
+      }
+      NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(1E300);
+      HyperparameterGradientsParameter.ActualValue = new RealVector(Hyperparameter.Count());
+      return base.Apply();
+    }
+
+    public static IGaussianProcessModel Create(IClassificationProblemData problemData, double[] hyperparameter, IMeanFunction meanFunction, ICovarianceFunction covarianceFunction, bool scaleInputs = true) {
+      return new GaussianProcessModel(problemData.Dataset, problemData.TargetVariable, problemData.AllowedInputVariables, problemData.TrainingIndices, hyperparameter, meanFunction, covarianceFunction, scaleInputs);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationSolutionCreator.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationSolutionCreator.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationSolutionCreator.cs	(revision 17041)
@@ -0,0 +1,120 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Operators;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("A2DE805A-046B-40C5-AB14-2AF633A2FB2F")]
+  [Item(Name = "GaussianProcessClassificationSolutionCreator",
+    Description = "Creates a Gaussian process solution from a trained model.")]
+  public sealed class GaussianProcessClassificationSolutionCreator : SingleSuccessorOperator {
+    private const string ProblemDataParameterName = "ProblemData";
+    private const string ModelParameterName = "GaussianProcessClassificationModel";
+    private const string SolutionParameterName = "Solution";
+    private const string ResultsParameterName = "Results";
+    private const string TrainingAccuracyResultName = "Accuracy (training)";
+    private const string TestAccuracyResultName = "Accuracy (test)";
+    private const string CreateSolutionParameterName = "CreateSolution";
+
+    #region Parameter Properties
+    public ILookupParameter<IClassificationProblemData> ProblemDataParameter {
+      get { return (ILookupParameter<IClassificationProblemData>)Parameters[ProblemDataParameterName]; }
+    }
+    public ILookupParameter<IDiscriminantFunctionClassificationSolution> SolutionParameter {
+      get { return (ILookupParameter<IDiscriminantFunctionClassificationSolution>)Parameters[SolutionParameterName]; }
+    }
+    public ILookupParameter<IGaussianProcessModel> ModelParameter {
+      get { return (ILookupParameter<IGaussianProcessModel>)Parameters[ModelParameterName]; }
+    }
+    public ILookupParameter<ResultCollection> ResultsParameter {
+      get { return (ILookupParameter<ResultCollection>)Parameters[ResultsParameterName]; }
+    }
+    public ILookupParameter<BoolValue> CreateSolutionParameter {
+      get { return (ILookupParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
+    }
+    #endregion
+
+    [StorableConstructor]
+    private GaussianProcessClassificationSolutionCreator(StorableConstructorFlag _) : base(_) { }
+    private GaussianProcessClassificationSolutionCreator(GaussianProcessClassificationSolutionCreator original, Cloner cloner) : base(original, cloner) { }
+    public GaussianProcessClassificationSolutionCreator()
+      : base() {
+      // in
+      Parameters.Add(new LookupParameter<IClassificationProblemData>(ProblemDataParameterName, "The classification problem data for the Gaussian process solution."));
+      Parameters.Add(new LookupParameter<IGaussianProcessModel>(ModelParameterName, "The Gaussian process classification model to use for the solution."));
+      Parameters.Add(new LookupParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run"));
+
+      // in & out
+      Parameters.Add(new LookupParameter<ResultCollection>(ResultsParameterName, "The result collection of the algorithm."));
+      // out
+      Parameters.Add(new LookupParameter<IDiscriminantFunctionClassificationSolution>(SolutionParameterName, "The produced Gaussian process solution."));
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
+      if (!Parameters.ContainsKey(CreateSolutionParameterName)) {
+        Parameters.Add(new LookupParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run"));
+      }
+      #endregion
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessClassificationSolutionCreator(this, cloner);
+    }
+
+    public override IOperation Apply() {
+      if (ModelParameter.ActualValue != null && CreateSolutionParameter.ActualValue.Value == true) {
+        var m = (IGaussianProcessModel)ModelParameter.ActualValue.Clone();
+        m.FixParameters();
+        var data = (IClassificationProblemData)ProblemDataParameter.ActualValue.Clone();
+        var model = new DiscriminantFunctionClassificationModel(m, new NormalDistributionCutPointsThresholdCalculator());
+        model.RecalculateModelParameters(data, data.TrainingIndices);
+        var s = model.CreateDiscriminantFunctionClassificationSolution(data);
+
+        SolutionParameter.ActualValue = s;
+        var results = ResultsParameter.ActualValue;
+        if (!results.ContainsKey(SolutionParameterName)) {
+          results.Add(new Result(SolutionParameterName, "The Gaussian process classification solution", s));
+          results.Add(new Result(TrainingAccuracyResultName,
+                                 "The accuracy of the Gaussian process solution on the training partition.",
+                                 new DoubleValue(s.TrainingAccuracy)));
+          results.Add(new Result(TestAccuracyResultName,
+                                 "The accuracy of the Gaussian process solution on the test partition.",
+                                 new DoubleValue(s.TestAccuracy)));
+        } else {
+          results[SolutionParameterName].Value = s;
+          results[TrainingAccuracyResultName].Value = new DoubleValue(s.TrainingAccuracy);
+          results[TestAccuracyResultName].Value = new DoubleValue(s.TestAccuracy);
+        }
+      }
+      return base.Apply();
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessCovarianceOptimizationProblem.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessCovarianceOptimizationProblem.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessCovarianceOptimizationProblem.cs	(revision 17041)
@@ -0,0 +1,424 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.Instances;
+
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("Gaussian Process Covariance Optimization Problem", "")]
+  [Creatable(CreatableAttribute.Categories.GeneticProgrammingProblems, Priority = 300)]
+  [StorableType("A3EA7CE7-78FA-48FF-9DD5-FBE5AB770A99")]
+  public sealed class GaussianProcessCovarianceOptimizationProblem : SymbolicExpressionTreeProblem, IStatefulItem, IRegressionProblem, IProblemInstanceConsumer<IRegressionProblemData>, IProblemInstanceExporter<IRegressionProblemData> {
+    #region static variables and ctor
+    private static readonly CovarianceMaternIso maternIso1;
+    private static readonly CovarianceMaternIso maternIso3;
+    private static readonly CovarianceMaternIso maternIso5;
+    private static readonly CovariancePiecewisePolynomial piecewisePoly0;
+    private static readonly CovariancePiecewisePolynomial piecewisePoly1;
+    private static readonly CovariancePiecewisePolynomial piecewisePoly2;
+    private static readonly CovariancePiecewisePolynomial piecewisePoly3;
+    private static readonly CovariancePolynomial poly2;
+    private static readonly CovariancePolynomial poly3;
+    private static readonly CovarianceSpectralMixture spectralMixture1;
+    private static readonly CovarianceSpectralMixture spectralMixture3;
+    private static readonly CovarianceSpectralMixture spectralMixture5;
+    private static readonly CovarianceLinear linear;
+    private static readonly CovarianceLinearArd linearArd;
+    private static readonly CovarianceNeuralNetwork neuralNetwork;
+    private static readonly CovariancePeriodic periodic;
+    private static readonly CovarianceRationalQuadraticIso ratQuadraticIso;
+    private static readonly CovarianceRationalQuadraticArd ratQuadraticArd;
+    private static readonly CovarianceSquaredExponentialArd sqrExpArd;
+    private static readonly CovarianceSquaredExponentialIso sqrExpIso;
+
+    static GaussianProcessCovarianceOptimizationProblem() {
+      // cumbersome initialization because of ConstrainedValueParameters
+      maternIso1 = new CovarianceMaternIso(); SetConstrainedValueParameter(maternIso1.DParameter, 1);
+      maternIso3 = new CovarianceMaternIso(); SetConstrainedValueParameter(maternIso3.DParameter, 3);
+      maternIso5 = new CovarianceMaternIso(); SetConstrainedValueParameter(maternIso5.DParameter, 5);
+
+      piecewisePoly0 = new CovariancePiecewisePolynomial(); SetConstrainedValueParameter(piecewisePoly0.VParameter, 0);
+      piecewisePoly1 = new CovariancePiecewisePolynomial(); SetConstrainedValueParameter(piecewisePoly1.VParameter, 1);
+      piecewisePoly2 = new CovariancePiecewisePolynomial(); SetConstrainedValueParameter(piecewisePoly2.VParameter, 2);
+      piecewisePoly3 = new CovariancePiecewisePolynomial(); SetConstrainedValueParameter(piecewisePoly3.VParameter, 3);
+
+      poly2 = new CovariancePolynomial(); poly2.DegreeParameter.Value.Value = 2;
+      poly3 = new CovariancePolynomial(); poly3.DegreeParameter.Value.Value = 3;
+
+      spectralMixture1 = new CovarianceSpectralMixture(); spectralMixture1.QParameter.Value.Value = 1;
+      spectralMixture3 = new CovarianceSpectralMixture(); spectralMixture3.QParameter.Value.Value = 3;
+      spectralMixture5 = new CovarianceSpectralMixture(); spectralMixture5.QParameter.Value.Value = 5;
+
+      linear = new CovarianceLinear();
+      linearArd = new CovarianceLinearArd();
+      neuralNetwork = new CovarianceNeuralNetwork();
+      periodic = new CovariancePeriodic();
+      ratQuadraticArd = new CovarianceRationalQuadraticArd();
+      ratQuadraticIso = new CovarianceRationalQuadraticIso();
+      sqrExpArd = new CovarianceSquaredExponentialArd();
+      sqrExpIso = new CovarianceSquaredExponentialIso();
+    }
+
+    private static void SetConstrainedValueParameter(IConstrainedValueParameter<IntValue> param, int val) {
+      param.Value = param.ValidValues.Single(v => v.Value == val);
+    }
+
+    #endregion
+
+    #region parameter names
+
+    private const string ProblemDataParameterName = "ProblemData";
+    private const string ConstantOptIterationsParameterName = "Constant optimization steps";
+    private const string RestartsParameterName = "Restarts";
+
+    #endregion
+
+    #region Parameter Properties
+    IParameter IDataAnalysisProblem.ProblemDataParameter { get { return ProblemDataParameter; } }
+
+    public IValueParameter<IRegressionProblemData> ProblemDataParameter {
+      get { return (IValueParameter<IRegressionProblemData>)Parameters[ProblemDataParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> ConstantOptIterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[ConstantOptIterationsParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> RestartsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[RestartsParameterName]; }
+    }
+    #endregion
+
+    #region Properties
+
+    public IRegressionProblemData ProblemData {
+      get { return ProblemDataParameter.Value; }
+      set { ProblemDataParameter.Value = value; }
+    }
+    IDataAnalysisProblemData IDataAnalysisProblem.ProblemData { get { return ProblemData; } }
+
+    public int ConstantOptIterations {
+      get { return ConstantOptIterationsParameter.Value.Value; }
+      set { ConstantOptIterationsParameter.Value.Value = value; }
+    }
+
+    public int Restarts {
+      get { return RestartsParameter.Value.Value; }
+      set { RestartsParameter.Value.Value = value; }
+    }
+    #endregion
+
+    public override bool Maximization {
+      get { return true; } // return log likelihood (instead of negative log likelihood as in GPR
+    }
+
+    // problem stores a few variables for information exchange from Evaluate() to Analyze()
+    private readonly object problemStateLocker = new object();
+    [Storable]
+    private double bestQ;
+    [Storable]
+    private double[] bestHyperParameters;
+    [Storable]
+    private IMeanFunction meanFunc;
+    [Storable]
+    private ICovarianceFunction covFunc;
+
+    public GaussianProcessCovarianceOptimizationProblem()
+      : base() {
+      Parameters.Add(new ValueParameter<IRegressionProblemData>(ProblemDataParameterName, "The data for the regression problem", new RegressionProblemData()));
+      Parameters.Add(new FixedValueParameter<IntValue>(ConstantOptIterationsParameterName, "Number of optimization steps for hyperparameter values", new IntValue(50)));
+      Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of random restarts for constant optimization.", new IntValue(10)));
+      Parameters["Restarts"].Hidden = true;
+      var g = new SimpleSymbolicExpressionGrammar();
+      g.AddSymbols(new string[] { "Sum", "Product" }, 2, 2);
+      g.AddTerminalSymbols(new string[]
+      {
+        "Linear",
+        "LinearArd",
+        "MaternIso1",
+        "MaternIso3",
+        "MaternIso5",
+        "NeuralNetwork",
+        "Periodic",
+        "PiecewisePolynomial0",
+        "PiecewisePolynomial1",
+        "PiecewisePolynomial2",
+        "PiecewisePolynomial3",
+        "Polynomial2",
+        "Polynomial3",
+        "RationalQuadraticArd",
+        "RationalQuadraticIso",
+        "SpectralMixture1",
+        "SpectralMixture3",
+        "SpectralMixture5",
+        "SquaredExponentialArd",
+        "SquaredExponentialIso"
+      });
+      base.Encoding = new SymbolicExpressionTreeEncoding(g, 10, 5);
+    }
+
+    public void InitializeState() { ClearState(); }
+    public void ClearState() {
+      meanFunc = null;
+      covFunc = null;
+      bestQ = double.NegativeInfinity;
+      bestHyperParameters = null;
+    }
+
+    private readonly object syncRoot = new object();
+    // Does not produce the same result for the same seed when using parallel engine (see below)!
+    public override double Evaluate(ISymbolicExpressionTree tree, IRandom random) {
+      var meanFunction = new MeanConst();
+      var problemData = ProblemData;
+      var ds = problemData.Dataset;
+      var targetVariable = problemData.TargetVariable;
+      var allowedInputVariables = problemData.AllowedInputVariables.ToArray();
+      var nVars = allowedInputVariables.Length;
+      var trainingRows = problemData.TrainingIndices.ToArray();
+
+      // use the same covariance function for each restart
+      var covarianceFunction = TreeToCovarianceFunction(tree);
+
+      // allocate hyperparameters
+      var hyperParameters = new double[meanFunction.GetNumberOfParameters(nVars) + covarianceFunction.GetNumberOfParameters(nVars) + 1]; // mean + cov + noise
+      double[] bestHyperParameters = new double[hyperParameters.Length];
+      var bestObjValue = new double[1] { double.MinValue };
+
+      // data that is necessary for the objective function
+      var data = Tuple.Create(ds, targetVariable, allowedInputVariables, trainingRows, (IMeanFunction)meanFunction, covarianceFunction, bestObjValue);
+
+      for (int t = 0; t < Restarts; t++) {
+        var prevBest = bestObjValue[0];
+        var prevBestHyperParameters = new double[hyperParameters.Length];
+        Array.Copy(bestHyperParameters, prevBestHyperParameters, bestHyperParameters.Length);
+
+        // initialize hyperparameters
+        hyperParameters[0] = ds.GetDoubleValues(targetVariable).Average(); // mean const
+
+        // Evaluate might be called concurrently therefore access to random has to be synchronized.
+        // However, results of multiple runs with the same seed will be different when using the parallel engine.
+        lock (syncRoot) {
+          for (int i = 0; i < covarianceFunction.GetNumberOfParameters(nVars); i++) {
+            hyperParameters[1 + i] = random.NextDouble() * 2.0 - 1.0;
+          }
+        }
+        hyperParameters[hyperParameters.Length - 1] = 1.0; // s² = exp(2), TODO: other inits better?
+
+        // use alglib.bfgs for hyper-parameter optimization ...
+        double epsg = 0;
+        double epsf = 0.00001;
+        double epsx = 0;
+        double stpmax = 1;
+        int maxits = ConstantOptIterations;
+        alglib.mincgstate state;
+        alglib.mincgreport rep;
+
+        alglib.mincgcreate(hyperParameters, out state);
+        alglib.mincgsetcond(state, epsg, epsf, epsx, maxits);
+        alglib.mincgsetstpmax(state, stpmax);
+        alglib.mincgoptimize(state, ObjectiveFunction, null, data);
+
+        alglib.mincgresults(state, out bestHyperParameters, out rep);
+
+        if (rep.terminationtype < 0) {
+          // error -> restore previous best quality
+          bestObjValue[0] = prevBest;
+          Array.Copy(prevBestHyperParameters, bestHyperParameters, prevBestHyperParameters.Length);
+        }
+      }
+
+      UpdateBestSoFar(bestObjValue[0], bestHyperParameters, meanFunction, covarianceFunction);
+
+      return bestObjValue[0];
+    }
+
+    // updates the overall best quality and overall best model for Analyze()
+    private void UpdateBestSoFar(double bestQ, double[] bestHyperParameters, IMeanFunction meanFunc, ICovarianceFunction covFunc) {
+      lock (problemStateLocker) {
+        if (bestQ > this.bestQ) {
+          this.bestQ = bestQ;
+          this.bestHyperParameters = new double[bestHyperParameters.Length];
+          Array.Copy(bestHyperParameters, this.bestHyperParameters, this.bestHyperParameters.Length);
+          this.meanFunc = meanFunc;
+          this.covFunc = covFunc;
+        }
+      }
+    }
+
+    public override void Analyze(ISymbolicExpressionTree[] trees, double[] qualities, ResultCollection results, IRandom random) {
+      if (!results.ContainsKey("Best Solution Quality")) {
+        results.Add(new Result("Best Solution Quality", typeof(DoubleValue)));
+      }
+      if (!results.ContainsKey("Best Tree")) {
+        results.Add(new Result("Best Tree", typeof(ISymbolicExpressionTree)));
+      }
+      if (!results.ContainsKey("Best Solution")) {
+        results.Add(new Result("Best Solution", typeof(GaussianProcessRegressionSolution)));
+      }
+
+      var bestQuality = qualities.Max();
+
+      if (results["Best Solution Quality"].Value == null || bestQuality > ((DoubleValue)results["Best Solution Quality"].Value).Value) {
+        var bestIdx = Array.IndexOf(qualities, bestQuality);
+        var bestClone = (ISymbolicExpressionTree)trees[bestIdx].Clone();
+        results["Best Tree"].Value = bestClone;
+        results["Best Solution Quality"].Value = new DoubleValue(bestQuality);
+        results["Best Solution"].Value = CreateSolution();
+      }
+    }
+
+    private IItem CreateSolution() {
+      var problemData = ProblemData;
+      var ds = problemData.Dataset;
+      var targetVariable = problemData.TargetVariable;
+      var allowedInputVariables = problemData.AllowedInputVariables.ToArray();
+      var trainingRows = problemData.TrainingIndices.ToArray();
+
+      lock (problemStateLocker) {
+        var model = new GaussianProcessModel(ds, targetVariable, allowedInputVariables, trainingRows, bestHyperParameters, (IMeanFunction)meanFunc.Clone(), (ICovarianceFunction)covFunc.Clone());
+        model.FixParameters();
+        return model.CreateRegressionSolution((IRegressionProblemData)ProblemData.Clone());
+      }
+    }
+
+    private void ObjectiveFunction(double[] x, ref double func, double[] grad, object obj) {
+      // we want to optimize the model likelihood by changing the hyperparameters and also return the gradient for each hyperparameter
+      var data = (Tuple<IDataset, string, string[], int[], IMeanFunction, ICovarianceFunction, double[]>)obj;
+      var ds = data.Item1;
+      var targetVariable = data.Item2;
+      var allowedInputVariables = data.Item3;
+      var trainingRows = data.Item4;
+      var meanFunction = data.Item5;
+      var covarianceFunction = data.Item6;
+      var bestObjValue = data.Item7;
+      var hyperParameters = x; // the decision variable vector
+
+      try {
+        var model = new GaussianProcessModel(ds, targetVariable, allowedInputVariables, trainingRows, hyperParameters, meanFunction, covarianceFunction);
+
+        func = model.NegativeLogLikelihood; // mincgoptimize, so we return negative likelihood
+        bestObjValue[0] = Math.Max(bestObjValue[0], -func); // problem itself is a maximization problem
+        var gradients = model.HyperparameterGradients;
+        Array.Copy(gradients, grad, gradients.Length);
+      }
+      catch (ArgumentException) {
+        // building the GaussianProcessModel might fail, in this case we return the worst possible objective value
+        func = 1.0E+300;
+        Array.Clear(grad, 0, grad.Length);
+      }
+    }
+
+    private ICovarianceFunction TreeToCovarianceFunction(ISymbolicExpressionTree tree) {
+      return TreeToCovarianceFunction(tree.Root.GetSubtree(0).GetSubtree(0)); // skip programroot and startsymbol
+    }
+
+    private ICovarianceFunction TreeToCovarianceFunction(ISymbolicExpressionTreeNode node) {
+      switch (node.Symbol.Name) {
+        case "Sum": {
+            var sum = new CovarianceSum();
+            sum.Terms.Add(TreeToCovarianceFunction(node.GetSubtree(0)));
+            sum.Terms.Add(TreeToCovarianceFunction(node.GetSubtree(1)));
+            return sum;
+          }
+        case "Product": {
+            var prod = new CovarianceProduct();
+            prod.Factors.Add(TreeToCovarianceFunction(node.GetSubtree(0)));
+            prod.Factors.Add(TreeToCovarianceFunction(node.GetSubtree(1)));
+            return prod;
+          }
+        // covFunction is cloned by the model so we can reuse instances of terminal covariance functions
+        case "Linear": return linear;
+        case "LinearArd": return linearArd;
+        case "MaternIso1": return maternIso1;
+        case "MaternIso3": return maternIso3;
+        case "MaternIso5": return maternIso5;
+        case "NeuralNetwork": return neuralNetwork;
+        case "Periodic": return periodic;
+        case "PiecewisePolynomial0": return piecewisePoly0;
+        case "PiecewisePolynomial1": return piecewisePoly1;
+        case "PiecewisePolynomial2": return piecewisePoly2;
+        case "PiecewisePolynomial3": return piecewisePoly3;
+        case "Polynomial2": return poly2;
+        case "Polynomial3": return poly3;
+        case "RationalQuadraticArd": return ratQuadraticArd;
+        case "RationalQuadraticIso": return ratQuadraticIso;
+        case "SpectralMixture1": return spectralMixture1;
+        case "SpectralMixture3": return spectralMixture3;
+        case "SpectralMixture5": return spectralMixture5;
+        case "SquaredExponentialArd": return sqrExpArd;
+        case "SquaredExponentialIso": return sqrExpIso;
+        default: throw new InvalidProgramException(string.Format("Found invalid symbol {0}", node.Symbol.Name));
+      }
+    }
+
+
+    // persistence
+    [StorableConstructor]
+    private GaussianProcessCovarianceOptimizationProblem(StorableConstructorFlag _) : base(_) { }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+    }
+
+    // cloning 
+    private GaussianProcessCovarianceOptimizationProblem(GaussianProcessCovarianceOptimizationProblem original, Cloner cloner)
+      : base(original, cloner) {
+      bestQ = original.bestQ;
+      meanFunc = cloner.Clone(original.meanFunc);
+      covFunc = cloner.Clone(original.covFunc);
+      if (bestHyperParameters != null) {
+        bestHyperParameters = new double[original.bestHyperParameters.Length];
+        Array.Copy(original.bestHyperParameters, bestHyperParameters, bestHyperParameters.Length);
+      }
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessCovarianceOptimizationProblem(this, cloner);
+    }
+
+    public void Load(IRegressionProblemData data) {
+      this.ProblemData = data;
+      OnProblemDataChanged();
+    }
+
+    public IRegressionProblemData Export() {
+      return ProblemData;
+    }
+
+    #region events
+    public event EventHandler ProblemDataChanged;
+
+
+    private void OnProblemDataChanged() {
+      var handler = ProblemDataChanged;
+      if (handler != null)
+        handler(this, EventArgs.Empty);
+    }
+    #endregion
+
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessHyperparameterInitializer.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessHyperparameterInitializer.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessHyperparameterInitializer.cs	(revision 17041)
@@ -0,0 +1,99 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Operators;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("73A3AC0C-849D-409F-8B5A-E5A08292A985")]
+  [Item(Name = "GaussianProcessHyperparameterInitializer",
+    Description = "Initializers the hyperparameter vector based on the mean function, covariance function, and number of allowed input variables.")]
+  public sealed class GaussianProcessHyperparameterInitializer : SingleSuccessorOperator {
+    private const string MeanFunctionParameterName = "MeanFunction";
+    private const string CovarianceFunctionParameterName = "CovarianceFunction";
+    private const string ProblemDataParameterName = "ProblemData";
+    private const string HyperparameterParameterName = "Hyperparameter";
+    private const string RandomParameterName = "Random";
+
+    #region Parameter Properties
+    // in
+    public ILookupParameter<IMeanFunction> MeanFunctionParameter {
+      get { return (ILookupParameter<IMeanFunction>)Parameters[MeanFunctionParameterName]; }
+    }
+    public ILookupParameter<ICovarianceFunction> CovarianceFunctionParameter {
+      get { return (ILookupParameter<ICovarianceFunction>)Parameters[CovarianceFunctionParameterName]; }
+    }
+    public ILookupParameter<IDataAnalysisProblemData> ProblemDataParameter {
+      get { return (ILookupParameter<IDataAnalysisProblemData>)Parameters[ProblemDataParameterName]; }
+    }
+    public ILookupParameter<IRandom> RandomParameter {
+      get { return (ILookupParameter<IRandom>)Parameters[RandomParameterName]; }
+    }
+    // out
+    public ILookupParameter<RealVector> HyperparameterParameter {
+      get { return (ILookupParameter<RealVector>)Parameters[HyperparameterParameterName]; }
+    }
+    #endregion
+
+    #region Properties
+    private IMeanFunction MeanFunction { get { return MeanFunctionParameter.ActualValue; } }
+    private ICovarianceFunction CovarianceFunction { get { return CovarianceFunctionParameter.ActualValue; } }
+    private IDataAnalysisProblemData ProblemData { get { return ProblemDataParameter.ActualValue; } }
+    #endregion
+
+    [StorableConstructor]
+    private GaussianProcessHyperparameterInitializer(StorableConstructorFlag _) : base(_) { }
+    private GaussianProcessHyperparameterInitializer(GaussianProcessHyperparameterInitializer original, Cloner cloner) : base(original, cloner) { }
+    public GaussianProcessHyperparameterInitializer()
+      : base() {
+      // in
+      Parameters.Add(new LookupParameter<IMeanFunction>(MeanFunctionParameterName, "The mean function for the Gaussian process model."));
+      Parameters.Add(new LookupParameter<ICovarianceFunction>(CovarianceFunctionParameterName, "The covariance function for the Gaussian process model."));
+      Parameters.Add(new LookupParameter<IDataAnalysisProblemData>(ProblemDataParameterName, "The input data for the Gaussian process."));
+      Parameters.Add(new LookupParameter<IRandom>(RandomParameterName, "The pseudo random number generator to use for initializing the hyperparameter vector."));
+      // out
+      Parameters.Add(new LookupParameter<RealVector>(HyperparameterParameterName, "The initial hyperparameter vector for the Gaussian process model."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessHyperparameterInitializer(this, cloner);
+    }
+
+    public override IOperation Apply() {
+      var inputVariablesCount = ProblemData.AllowedInputVariables.Count();
+      int l = 1 + MeanFunction.GetNumberOfParameters(inputVariablesCount) +
+              CovarianceFunction.GetNumberOfParameters(inputVariablesCount);
+      var r = new RealVector(l);
+      var rand = RandomParameter.ActualValue;
+      for (int i = 0; i < r.Length; i++)
+        r[i] = rand.NextDouble() * 10 - 5;
+
+      HyperparameterParameter.ActualValue = r;
+      return base.Apply();
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs	(revision 17041)
@@ -0,0 +1,431 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a Gaussian process model.
+  /// </summary>
+  [StorableType("37B5DC24-D6BB-4DA9-9A08-ACBB0ED1A9E9")]
+  [Item("GaussianProcessModel", "Represents a Gaussian process posterior.")]
+  public sealed class GaussianProcessModel : RegressionModel, IGaussianProcessModel {
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return allowedInputVariables; }
+    }
+
+    [Storable]
+    private double negativeLogLikelihood;
+    public double NegativeLogLikelihood {
+      get { return negativeLogLikelihood; }
+    }
+
+    [Storable]
+    private double loocvNegLogPseudoLikelihood;
+    public double LooCvNegativeLogPseudoLikelihood {
+      get { return loocvNegLogPseudoLikelihood; }
+    }
+
+    [Storable]
+    private double[] hyperparameterGradients;
+    public double[] HyperparameterGradients {
+      get {
+        var copy = new double[hyperparameterGradients.Length];
+        Array.Copy(hyperparameterGradients, copy, copy.Length);
+        return copy;
+      }
+    }
+
+    [Storable]
+    private ICovarianceFunction covarianceFunction;
+    public ICovarianceFunction CovarianceFunction {
+      get { return covarianceFunction; }
+    }
+    [Storable]
+    private IMeanFunction meanFunction;
+    public IMeanFunction MeanFunction {
+      get { return meanFunction; }
+    }
+
+    [Storable]
+    private string[] allowedInputVariables;
+    public string[] AllowedInputVariables {
+      get { return allowedInputVariables; }
+    }
+
+    [Storable]
+    private double[] alpha;
+    [Storable]
+    private double sqrSigmaNoise;
+    public double SigmaNoise {
+      get { return Math.Sqrt(sqrSigmaNoise); }
+    }
+
+    [Storable]
+    private double[] meanParameter;
+    [Storable]
+    private double[] covarianceParameter;
+
+    private double[,] l; // used to be storable in previous versions (is calculated lazily now)
+    private double[,] x; // scaled training dataset, used to be storable in previous versions (is calculated lazily now)
+
+    // BackwardsCompatibility3.4
+    #region Backwards compatible code, remove with 3.5
+    [Storable(Name = "l")] // restore if available but don't store anymore
+    private double[,] l_storable {
+      set { this.l = value; }
+      get {
+        if (trainingDataset == null) return l; // this model has been created with an old version 
+        else return null; // if the training dataset is available l should not be serialized
+      }
+    }
+    [Storable(Name = "x")] // restore if available but don't store anymore
+    private double[,] x_storable {
+      set { this.x = value; }
+      get {
+        if (trainingDataset == null) return x; // this model has been created with an old version 
+        else return null; // if the training dataset is available x should not be serialized
+      }
+    }
+    #endregion
+
+
+    [Storable]
+    private IDataset trainingDataset; // it is better to store the original training dataset completely because this is more efficient in persistence
+    [Storable]
+    private int[] trainingRows;
+
+    [Storable]
+    private Scaling inputScaling;
+
+
+    [StorableConstructor]
+    private GaussianProcessModel(StorableConstructorFlag _) : base(_) { }
+    private GaussianProcessModel(GaussianProcessModel original, Cloner cloner)
+      : base(original, cloner) {
+      this.meanFunction = cloner.Clone(original.meanFunction);
+      this.covarianceFunction = cloner.Clone(original.covarianceFunction);
+      if (original.inputScaling != null)
+        this.inputScaling = cloner.Clone(original.inputScaling);
+      this.trainingDataset = cloner.Clone(original.trainingDataset);
+      this.negativeLogLikelihood = original.negativeLogLikelihood;
+      this.loocvNegLogPseudoLikelihood = original.loocvNegLogPseudoLikelihood;
+      this.sqrSigmaNoise = original.sqrSigmaNoise;
+      if (original.meanParameter != null) {
+        this.meanParameter = (double[])original.meanParameter.Clone();
+      }
+      if (original.covarianceParameter != null) {
+        this.covarianceParameter = (double[])original.covarianceParameter.Clone();
+      }
+
+      // shallow copies of arrays because they cannot be modified
+      this.trainingRows = original.trainingRows;
+      this.allowedInputVariables = original.allowedInputVariables;
+      this.alpha = original.alpha;
+      this.l = original.l;
+      this.x = original.x;
+    }
+    public GaussianProcessModel(IDataset ds, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<int> rows,
+      IEnumerable<double> hyp, IMeanFunction meanFunction, ICovarianceFunction covarianceFunction,
+      bool scaleInputs = true)
+      : base(targetVariable) {
+      this.name = ItemName;
+      this.description = ItemDescription;
+      this.meanFunction = (IMeanFunction)meanFunction.Clone();
+      this.covarianceFunction = (ICovarianceFunction)covarianceFunction.Clone();
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+
+
+      int nVariables = this.allowedInputVariables.Length;
+      meanParameter = hyp
+        .Take(this.meanFunction.GetNumberOfParameters(nVariables))
+        .ToArray();
+
+      covarianceParameter = hyp.Skip(this.meanFunction.GetNumberOfParameters(nVariables))
+                                             .Take(this.covarianceFunction.GetNumberOfParameters(nVariables))
+                                             .ToArray();
+      sqrSigmaNoise = Math.Exp(2.0 * hyp.Last());
+      try {
+        CalculateModel(ds, rows, scaleInputs);
+      } catch (alglib.alglibexception ae) {
+        // wrap exception so that calling code doesn't have to know about alglib implementation
+        throw new ArgumentException("There was a problem in the calculation of the Gaussian process model", ae);
+      }
+    }
+
+    private void CalculateModel(IDataset ds, IEnumerable<int> rows, bool scaleInputs = true) {
+      this.trainingDataset = (IDataset)ds.Clone();
+      this.trainingRows = rows.ToArray();
+      this.inputScaling = scaleInputs ? new Scaling(ds, allowedInputVariables, rows) : null;
+
+      x = GetData(ds, this.allowedInputVariables, this.trainingRows, this.inputScaling);
+
+      IEnumerable<double> y;
+      y = ds.GetDoubleValues(TargetVariable, rows);
+
+      int n = x.GetLength(0);
+
+      var columns = Enumerable.Range(0, x.GetLength(1)).ToArray();
+      // calculate cholesky decomposed (lower triangular) covariance matrix
+      var cov = covarianceFunction.GetParameterizedCovarianceFunction(covarianceParameter, columns);
+      this.l = CalculateL(x, cov, sqrSigmaNoise);
+
+      // calculate mean
+      var mean = meanFunction.GetParameterizedMeanFunction(meanParameter, columns);
+      double[] m = Enumerable.Range(0, x.GetLength(0))
+        .Select(r => mean.Mean(x, r))
+        .ToArray();
+
+      // calculate sum of diagonal elements for likelihood
+      double diagSum = Enumerable.Range(0, n).Select(i => Math.Log(l[i, i])).Sum();
+
+      // solve for alpha
+      double[] ym = y.Zip(m, (a, b) => a - b).ToArray();
+
+      int info;
+      alglib.densesolverreport denseSolveRep;
+
+      alglib.spdmatrixcholeskysolve(l, n, false, ym, out info, out denseSolveRep, out alpha);
+      for (int i = 0; i < alpha.Length; i++)
+        alpha[i] = alpha[i] / sqrSigmaNoise;
+      negativeLogLikelihood = 0.5 * Util.ScalarProd(ym, alpha) + diagSum + (n / 2.0) * Math.Log(2.0 * Math.PI * sqrSigmaNoise);
+
+      // derivatives
+      int nAllowedVariables = x.GetLength(1);
+
+      alglib.matinvreport matInvRep;
+      double[,] lCopy = new double[l.GetLength(0), l.GetLength(1)];
+      Array.Copy(l, lCopy, lCopy.Length);
+
+      alglib.spdmatrixcholeskyinverse(ref lCopy, n, false, out info, out matInvRep);
+      if (info != 1) throw new ArgumentException("Can't invert matrix to calculate gradients.");
+
+      // LOOCV log pseudo-likelihood (or log predictive probability) (GPML page 116 and 117)
+      var sumLoo = 0.0;
+      var ki = new double[n];
+      for (int i = 0; i < n; i++) {
+        for (int j = 0; j < n; j++) ki[j] = cov.Covariance(x, i, j);
+        ki[i] += sqrSigmaNoise;
+
+        var yi = Util.ScalarProd(ki, alpha);
+        var yi_loo = yi - alpha[i] / (lCopy[i, i] / sqrSigmaNoise);
+        var s2_loo = 1.0 / (lCopy[i, i] / sqrSigmaNoise);
+        var err = ym[i] - yi_loo;
+        var nll_loo = 0.5 * Math.Log(2 * Math.PI * s2_loo) + 0.5 * err * err / s2_loo;
+        sumLoo += nll_loo;
+      }
+      loocvNegLogPseudoLikelihood = sumLoo;
+
+      for (int i = 0; i < n; i++) {
+        for (int j = 0; j <= i; j++)
+          lCopy[i, j] = lCopy[i, j] / sqrSigmaNoise - alpha[i] * alpha[j];
+      }
+
+      double noiseGradient = sqrSigmaNoise * Enumerable.Range(0, n).Select(i => lCopy[i, i]).Sum();
+
+      double[] meanGradients = new double[meanFunction.GetNumberOfParameters(nAllowedVariables)];
+      for (int k = 0; k < meanGradients.Length; k++) {
+        var meanGrad = new double[alpha.Length];
+        for (int g = 0; g < meanGrad.Length; g++)
+          meanGrad[g] = mean.Gradient(x, g, k);
+        meanGradients[k] = -Util.ScalarProd(meanGrad, alpha);
+      }
+
+      double[] covGradients = new double[covarianceFunction.GetNumberOfParameters(nAllowedVariables)];
+      if (covGradients.Length > 0) {
+        for (int i = 0; i < n; i++) {
+          for (int j = 0; j < i; j++) {
+            var g = cov.CovarianceGradient(x, i, j);
+            for (int k = 0; k < covGradients.Length; k++) {
+              covGradients[k] += lCopy[i, j] * g[k];
+            }
+          }
+
+          var gDiag = cov.CovarianceGradient(x, i, i);
+          for (int k = 0; k < covGradients.Length; k++) {
+            // diag
+            covGradients[k] += 0.5 * lCopy[i, i] * gDiag[k];
+          }
+        }
+      }
+
+      hyperparameterGradients =
+        meanGradients
+        .Concat(covGradients)
+        .Concat(new double[] { noiseGradient }).ToArray();
+
+    }
+
+    private static double[,] GetData(IDataset ds, IEnumerable<string> allowedInputs, IEnumerable<int> rows, Scaling scaling) {
+      if (scaling != null) {
+        // BackwardsCompatibility3.3
+        #region Backwards compatible code, remove with 3.4
+        // TODO: completely remove Scaling class
+        List<string> variablesList = allowedInputs.ToList();
+        List<int> rowsList = rows.ToList();
+
+        double[,] matrix = new double[rowsList.Count, variablesList.Count];
+
+        int col = 0;
+        foreach (string column in variablesList) {
+          var values = scaling.GetScaledValues(ds, column, rowsList);
+          int row = 0;
+          foreach (var value in values) {
+            matrix[row, col] = value;
+            row++;
+          }
+          col++;
+        }
+        return matrix;
+        #endregion
+      } else {
+        return ds.ToArray(allowedInputs, rows);
+      }
+    }
+
+    private static double[,] CalculateL(double[,] x, ParameterizedCovarianceFunction cov, double sqrSigmaNoise) {
+      int n = x.GetLength(0);
+      var l = new double[n, n];
+
+      // calculate covariances
+      for (int i = 0; i < n; i++) {
+        for (int j = i; j < n; j++) {
+          l[j, i] = cov.Covariance(x, i, j) / sqrSigmaNoise;
+          if (j == i) l[j, i] += 1.0;
+        }
+      }
+
+      // cholesky decomposition
+      var res = alglib.trfac.spdmatrixcholesky(ref l, n, false);
+      if (!res) throw new ArgumentException("Matrix is not positive semidefinite");
+      return l;
+    }
+
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessModel(this, cloner);
+    }
+
+    // is called by the solution creator to set all parameter values of the covariance and mean function
+    // to the optimized values (necessary to make the values visible in the GUI)
+    public void FixParameters() {
+      covarianceFunction.SetParameter(covarianceParameter);
+      meanFunction.SetParameter(meanParameter);
+      covarianceParameter = new double[0];
+      meanParameter = new double[0];
+    }
+
+    #region IRegressionModel Members
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      return GetEstimatedValuesHelper(dataset, rows);
+    }
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new GaussianProcessRegressionSolution(this, new RegressionProblemData(problemData));
+    }
+    #endregion
+
+
+    private IEnumerable<double> GetEstimatedValuesHelper(IDataset dataset, IEnumerable<int> rows) {
+      try {
+        if (x == null) {
+          x = GetData(trainingDataset, allowedInputVariables, trainingRows, inputScaling);
+        }
+        int n = x.GetLength(0);
+
+        double[,] newX = GetData(dataset, allowedInputVariables, rows, inputScaling);
+        int newN = newX.GetLength(0);
+
+        var Ks = new double[newN][];
+        var columns = Enumerable.Range(0, newX.GetLength(1)).ToArray();
+        var mean = meanFunction.GetParameterizedMeanFunction(meanParameter, columns);
+        var ms = Enumerable.Range(0, newX.GetLength(0))
+        .Select(r => mean.Mean(newX, r))
+        .ToArray();
+        var cov = covarianceFunction.GetParameterizedCovarianceFunction(covarianceParameter, columns);
+        for (int i = 0; i < newN; i++) {
+          Ks[i] = new double[n];
+          for (int j = 0; j < n; j++) {
+            Ks[i][j] = cov.CrossCovariance(x, newX, j, i);
+          }
+        }
+
+        return Enumerable.Range(0, newN)
+          .Select(i => ms[i] + Util.ScalarProd(Ks[i], alpha));
+      } catch (alglib.alglibexception ae) {
+        // wrap exception so that calling code doesn't have to know about alglib implementation
+        throw new ArgumentException("There was a problem in the calculation of the Gaussian process model", ae);
+      }
+    }
+
+    public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
+      try {
+        if (x == null) {
+          x = GetData(trainingDataset, allowedInputVariables, trainingRows, inputScaling);
+        }
+        int n = x.GetLength(0);
+
+        var newX = GetData(dataset, allowedInputVariables, rows, inputScaling);
+        int newN = newX.GetLength(0);
+
+        var kss = new double[newN];
+        double[,] sWKs = new double[n, newN];
+        var columns = Enumerable.Range(0, newX.GetLength(1)).ToArray();
+        var cov = covarianceFunction.GetParameterizedCovarianceFunction(covarianceParameter, columns);
+
+        if (l == null) {
+          l = CalculateL(x, cov, sqrSigmaNoise);
+        }
+
+        // for stddev 
+        for (int i = 0; i < newN; i++)
+          kss[i] = cov.Covariance(newX, i, i);
+
+        for (int i = 0; i < newN; i++) {
+          for (int j = 0; j < n; j++) {
+            sWKs[j, i] = cov.CrossCovariance(x, newX, j, i) / Math.Sqrt(sqrSigmaNoise);
+          }
+        }
+
+        // for stddev 
+        alglib.ablas.rmatrixlefttrsm(n, newN, l, 0, 0, false, false, 0, ref sWKs, 0, 0);
+
+        for (int i = 0; i < newN; i++) {
+          var col = Util.GetCol(sWKs, i).ToArray();
+          var sumV = Util.ScalarProd(col, col);
+          kss[i] += sqrSigmaNoise; // kss is V(f), add noise variance of predictive distibution to get V(y)
+          kss[i] -= sumV;
+          if (kss[i] < 0) kss[i] = 0;
+        }
+        return kss;
+      } catch (alglib.alglibexception ae) {
+        // wrap exception so that calling code doesn't have to know about alglib implementation
+        throw new ArgumentException("There was a problem in the calculation of the Gaussian process model", ae);
+      }
+    }
+
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModelCreator.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModelCreator.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModelCreator.cs	(revision 17041)
@@ -0,0 +1,113 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Operators;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("709D0C3B-EFDC-4113-B009-E0CE95A835F6")]
+  // base class for GaussianProcessModelCreators (specific for classification and regression)
+  public abstract class GaussianProcessModelCreator : SingleSuccessorOperator {
+    private const string HyperparameterParameterName = "Hyperparameter";
+    private const string MeanFunctionParameterName = "MeanFunction";
+    private const string CovarianceFunctionParameterName = "CovarianceFunction";
+    private const string ModelParameterName = "Model";
+    private const string NegativeLogLikelihoodParameterName = "NegativeLogLikelihood";
+    private const string NegativeLogPseudoLikelihoodParameterName = "NegativeLogPseudoLikelihood (LOOCV)";
+    private const string HyperparameterGradientsParameterName = "HyperparameterGradients";
+    protected const string ScaleInputValuesParameterName = "ScaleInputValues";
+
+    #region Parameter Properties
+    // in
+    public ILookupParameter<RealVector> HyperparameterParameter {
+      get { return (ILookupParameter<RealVector>)Parameters[HyperparameterParameterName]; }
+    }
+    public ILookupParameter<IMeanFunction> MeanFunctionParameter {
+      get { return (ILookupParameter<IMeanFunction>)Parameters[MeanFunctionParameterName]; }
+    }
+    public ILookupParameter<ICovarianceFunction> CovarianceFunctionParameter {
+      get { return (ILookupParameter<ICovarianceFunction>)Parameters[CovarianceFunctionParameterName]; }
+    }
+    // out
+    public ILookupParameter<IGaussianProcessModel> ModelParameter {
+      get { return (ILookupParameter<IGaussianProcessModel>)Parameters[ModelParameterName]; }
+    }
+    public ILookupParameter<RealVector> HyperparameterGradientsParameter {
+      get { return (ILookupParameter<RealVector>)Parameters[HyperparameterGradientsParameterName]; }
+    }
+    public ILookupParameter<DoubleValue> NegativeLogLikelihoodParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters[NegativeLogLikelihoodParameterName]; }
+    }
+    public ILookupParameter<DoubleValue> NegativeLogPseudoLikelihoodParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters[NegativeLogPseudoLikelihoodParameterName]; }
+    }
+    public ILookupParameter<BoolValue> ScaleInputValuesParameter {
+      get { return (ILookupParameter<BoolValue>)Parameters[ScaleInputValuesParameterName]; }
+    }
+    #endregion
+
+    #region Properties
+    protected RealVector Hyperparameter { get { return HyperparameterParameter.ActualValue; } }
+    protected IMeanFunction MeanFunction { get { return MeanFunctionParameter.ActualValue; } }
+    protected ICovarianceFunction CovarianceFunction { get { return CovarianceFunctionParameter.ActualValue; } }
+    public bool ScaleInputValues { get { return ScaleInputValuesParameter.ActualValue.Value; } }
+    #endregion
+
+    [StorableConstructor]
+    protected GaussianProcessModelCreator(StorableConstructorFlag _) : base(_) { }
+    protected GaussianProcessModelCreator(GaussianProcessModelCreator original, Cloner cloner) : base(original, cloner) { }
+    protected GaussianProcessModelCreator()
+      : base() {
+      // in
+      Parameters.Add(new LookupParameter<RealVector>(HyperparameterParameterName, "The hyperparameters for the Gaussian process model."));
+      Parameters.Add(new LookupParameter<IMeanFunction>(MeanFunctionParameterName, "The mean function for the Gaussian process model."));
+      Parameters.Add(new LookupParameter<ICovarianceFunction>(CovarianceFunctionParameterName, "The covariance function for the Gaussian process model."));
+      // out
+      Parameters.Add(new LookupParameter<IGaussianProcessModel>(ModelParameterName, "The resulting Gaussian process model"));
+      Parameters.Add(new LookupParameter<RealVector>(HyperparameterGradientsParameterName, "The gradients of the hyperparameters for the produced Gaussian process model (necessary for hyperparameter optimization)"));
+      Parameters.Add(new LookupParameter<DoubleValue>(NegativeLogLikelihoodParameterName, "The negative log-likelihood of the produced Gaussian process model given the data."));
+      Parameters.Add(new LookupParameter<DoubleValue>(NegativeLogPseudoLikelihoodParameterName, "The leave-one-out-cross-validation negative log pseudo-likelihood of the produced Gaussian process model given the data."));
+
+
+      Parameters.Add(new LookupParameter<BoolValue>(ScaleInputValuesParameterName,
+        "Determines if the input variable values are scaled to the range [0..1] for training."));
+      Parameters[ScaleInputValuesParameterName].Hidden = true;
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      if (!Parameters.ContainsKey(ScaleInputValuesParameterName)) {
+        Parameters.Add(new LookupParameter<BoolValue>(ScaleInputValuesParameterName,
+          "Determines if the input variable values are scaled to the range [0..1] for training."));
+        Parameters[ScaleInputValuesParameterName].Hidden = true;
+      }
+      if (!Parameters.ContainsKey(NegativeLogPseudoLikelihoodParameterName)) {
+        Parameters.Add(new LookupParameter<DoubleValue>(NegativeLogPseudoLikelihoodParameterName,
+          "The leave-one-out-cross-validation negative log pseudo-likelihood of the produced Gaussian process model given the data."));
+      }
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegression.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegression.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegression.cs	(revision 17041)
@@ -0,0 +1,154 @@
+
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.PluginInfrastructure;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  ///Gaussian process regression data analysis algorithm.
+  /// </summary>
+  [Item("Gaussian Process Regression", "Gaussian process regression data analysis algorithm.")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 160)]
+  [StorableType("3A1DF0A2-66D1-47BB-8BE4-23003BD34271")]
+  public sealed class GaussianProcessRegression : GaussianProcessBase, IStorableContent, IDataAnalysisAlgorithm<IRegressionProblem> {
+    public string Filename { get; set; }
+
+    public override Type ProblemType { get { return typeof(IRegressionProblem); } }
+    public new IRegressionProblem Problem {
+      get { return (IRegressionProblem)base.Problem; }
+      set { base.Problem = value; }
+    }
+
+    private const string ModelParameterName = "Model";
+    private const string CreateSolutionParameterName = "CreateSolution";
+
+
+    #region parameter properties
+    public IConstrainedValueParameter<IGaussianProcessRegressionModelCreator> GaussianProcessModelCreatorParameter {
+      get { return (IConstrainedValueParameter<IGaussianProcessRegressionModelCreator>)Parameters[ModelCreatorParameterName]; }
+    }
+    public IFixedValueParameter<GaussianProcessRegressionSolutionCreator> GaussianProcessSolutionCreatorParameter {
+      get { return (IFixedValueParameter<GaussianProcessRegressionSolutionCreator>)Parameters[SolutionCreatorParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> CreateSolutionParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
+    }
+    #endregion
+    #region properties
+    public bool CreateSolution {
+      get { return CreateSolutionParameter.Value.Value; }
+      set { CreateSolutionParameter.Value.Value = value; }
+    }
+    #endregion
+
+    [StorableConstructor]
+    private GaussianProcessRegression(StorableConstructorFlag _) : base(_) { }
+    private GaussianProcessRegression(GaussianProcessRegression original, Cloner cloner)
+      : base(original, cloner) {
+      RegisterEventHandlers();
+    }
+    public GaussianProcessRegression()
+      : base(new RegressionProblem()) {
+      this.name = ItemName;
+      this.description = ItemDescription;
+
+      var modelCreators = ApplicationManager.Manager.GetInstances<IGaussianProcessRegressionModelCreator>();
+      var defaultModelCreator = modelCreators.First(c => c is GaussianProcessRegressionModelCreator);
+
+      // GP regression and classification algorithms only differ in the model and solution creators,
+      // thus we use a common base class and use operator parameters to implement the specific versions.
+      // Different model creators can be implemented,
+      // but the solution creator is implemented in a generic fashion already and we don't allow derived solution creators
+      Parameters.Add(new ConstrainedValueParameter<IGaussianProcessRegressionModelCreator>(ModelCreatorParameterName, "The operator to create the Gaussian process model.",
+        new ItemSet<IGaussianProcessRegressionModelCreator>(modelCreators), defaultModelCreator));
+      // the solution creator cannot be changed
+      Parameters.Add(new FixedValueParameter<GaussianProcessRegressionSolutionCreator>(SolutionCreatorParameterName, "The solution creator for the algorithm",
+        new GaussianProcessRegressionSolutionCreator()));
+      Parameters[SolutionCreatorParameterName].Hidden = true;
+      // TODO: it would be better to deactivate the solution creator when this parameter is changed
+      Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+      Parameters[CreateSolutionParameterName].Hidden = true;
+
+      ParameterizedModelCreators();
+      ParameterizeSolutionCreator(GaussianProcessSolutionCreatorParameter.Value);
+      RegisterEventHandlers();
+    }
+
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
+      if (!Parameters.ContainsKey(CreateSolutionParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+        Parameters[CreateSolutionParameterName].Hidden = true;
+      }
+      #endregion
+      RegisterEventHandlers();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessRegression(this, cloner);
+    }
+
+    #region events
+    private void RegisterEventHandlers() {
+      GaussianProcessModelCreatorParameter.ValueChanged += ModelCreatorParameter_ValueChanged;
+    }
+
+    private void ModelCreatorParameter_ValueChanged(object sender, EventArgs e) {
+      ParameterizedModelCreator(GaussianProcessModelCreatorParameter.Value);
+    }
+    #endregion
+
+    private void ParameterizedModelCreators() {
+      foreach (var creator in GaussianProcessModelCreatorParameter.ValidValues) {
+        ParameterizedModelCreator(creator);
+      }
+    }
+
+    private void ParameterizedModelCreator(IGaussianProcessRegressionModelCreator modelCreator) {
+      modelCreator.ProblemDataParameter.ActualName = Problem.ProblemDataParameter.Name;
+      modelCreator.MeanFunctionParameter.ActualName = MeanFunctionParameterName;
+      modelCreator.CovarianceFunctionParameter.ActualName = CovarianceFunctionParameterName;
+
+      // parameter names fixed by the algorithm
+      modelCreator.ModelParameter.ActualName = ModelParameterName;
+      modelCreator.HyperparameterParameter.ActualName = HyperparameterParameterName;
+      modelCreator.HyperparameterGradientsParameter.ActualName = HyperparameterGradientsParameterName;
+      modelCreator.NegativeLogLikelihoodParameter.ActualName = NegativeLogLikelihoodParameterName;
+    }
+
+    private void ParameterizeSolutionCreator(GaussianProcessRegressionSolutionCreator solutionCreator) {
+      solutionCreator.ModelParameter.ActualName = ModelParameterName;
+      solutionCreator.ProblemDataParameter.ActualName = Problem.ProblemDataParameter.Name;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs	(revision 17041)
@@ -0,0 +1,82 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("0C20B1D2-9A58-4E77-9300-F5D76650DC19")]
+  [Item(Name = "GaussianProcessRegressionModelCreator",
+    Description = "Creates a Gaussian process model for regression given the data, the hyperparameters, a mean function, and a covariance function.")]
+  public sealed class GaussianProcessRegressionModelCreator : GaussianProcessModelCreator, IGaussianProcessRegressionModelCreator {
+    private const string ProblemDataParameterName = "ProblemData";
+
+    #region Parameter Properties
+    public ILookupParameter<IRegressionProblemData> ProblemDataParameter {
+      get { return (ILookupParameter<IRegressionProblemData>)Parameters[ProblemDataParameterName]; }
+    }
+    #endregion
+
+    #region Properties
+    private IRegressionProblemData ProblemData {
+      get { return ProblemDataParameter.ActualValue; }
+    }
+    #endregion
+    [StorableConstructor]
+    private GaussianProcessRegressionModelCreator(StorableConstructorFlag _) : base(_) { }
+    private GaussianProcessRegressionModelCreator(GaussianProcessRegressionModelCreator original, Cloner cloner) : base(original, cloner) { }
+    public GaussianProcessRegressionModelCreator()
+      : base() {
+      Parameters.Add(new LookupParameter<IRegressionProblemData>(ProblemDataParameterName, "The regression problem data for the Gaussian process model."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessRegressionModelCreator(this, cloner);
+    }
+
+    public override IOperation Apply() {
+      try {
+        var model = Create(ProblemData, Hyperparameter.ToArray(), MeanFunction, CovarianceFunction, ScaleInputValues);
+        ModelParameter.ActualValue = model;
+        NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(model.NegativeLogLikelihood);
+        NegativeLogPseudoLikelihoodParameter.ActualValue = new DoubleValue(model.LooCvNegativeLogPseudoLikelihood);
+        HyperparameterGradientsParameter.ActualValue = new RealVector(model.HyperparameterGradients);
+        return base.Apply();
+      }
+      catch (ArgumentException) { }
+      catch (alglib.alglibexception) { }
+      NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(1E300);
+      HyperparameterGradientsParameter.ActualValue = new RealVector(Hyperparameter.Count());
+      return base.Apply();
+    }
+
+    public static IGaussianProcessModel Create(IRegressionProblemData problemData, double[] hyperparameter, IMeanFunction meanFunction, ICovarianceFunction covarianceFunction, bool scaleInputs = true) {
+      return new GaussianProcessModel(problemData.Dataset, problemData.TargetVariable, problemData.AllowedInputVariables, problemData.TrainingIndices, hyperparameter, meanFunction, covarianceFunction, scaleInputs);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolution.cs	(revision 17041)
@@ -0,0 +1,52 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a Gaussian process solution for a regression problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("GaussianProcessRegressionSolution", "Represents a Gaussian process solution for a regression problem which can be visualized in the GUI.")]
+  [StorableType("52EDA00F-1340-49C5-B209-99182BBED559")]
+  public sealed class GaussianProcessRegressionSolution : ConfidenceRegressionSolution, IGaussianProcessSolution {
+
+    public new IGaussianProcessModel Model {
+      get { return (IGaussianProcessModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private GaussianProcessRegressionSolution(StorableConstructorFlag _) : base(_) {
+    }
+    private GaussianProcessRegressionSolution(GaussianProcessRegressionSolution original, Cloner cloner)
+      : base(original, cloner) { }
+    public GaussianProcessRegressionSolution(IGaussianProcessModel model, IRegressionProblemData problemData)
+      : base(model, problemData) { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessRegressionSolution(this, cloner);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs	(revision 17041)
@@ -0,0 +1,124 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Operators;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("9255DA02-C545-4952-AB36-5A588EE41407")]
+  [Item(Name = "GaussianProcessRegressionSolutionCreator",
+    Description = "Creates a Gaussian process solution from a trained model.")]
+  public sealed class GaussianProcessRegressionSolutionCreator : SingleSuccessorOperator {
+    private const string ProblemDataParameterName = "ProblemData";
+    private const string ModelParameterName = "GaussianProcessRegressionModel";
+    private const string SolutionParameterName = "Solution";
+    private const string ResultsParameterName = "Results";
+    private const string TrainingRSquaredResultName = "Training R²";
+    private const string TestRSquaredResultName = "Test R²";
+    private const string CreateSolutionParameterName = "CreateSolution";
+    private const string NegLogPseudoLikelihood = "Negative log pseudo-likelihood (LOO-CV)";
+
+    #region Parameter Properties
+    public ILookupParameter<IRegressionProblemData> ProblemDataParameter {
+      get { return (ILookupParameter<IRegressionProblemData>)Parameters[ProblemDataParameterName]; }
+    }
+    public ILookupParameter<IGaussianProcessSolution> SolutionParameter {
+      get { return (ILookupParameter<IGaussianProcessSolution>)Parameters[SolutionParameterName]; }
+    }
+    public ILookupParameter<IGaussianProcessModel> ModelParameter {
+      get { return (ILookupParameter<IGaussianProcessModel>)Parameters[ModelParameterName]; }
+    }
+    public ILookupParameter<ResultCollection> ResultsParameter {
+      get { return (ILookupParameter<ResultCollection>)Parameters[ResultsParameterName]; }
+    }
+    public ILookupParameter<BoolValue> CreateSolutionParameter {
+      get { return (ILookupParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
+    }
+    #endregion
+
+    [StorableConstructor]
+    private GaussianProcessRegressionSolutionCreator(StorableConstructorFlag _) : base(_) { }
+    private GaussianProcessRegressionSolutionCreator(GaussianProcessRegressionSolutionCreator original, Cloner cloner) : base(original, cloner) { }
+    public GaussianProcessRegressionSolutionCreator()
+      : base() {
+      // in
+      Parameters.Add(new LookupParameter<IRegressionProblemData>(ProblemDataParameterName, "The regression problem data for the Gaussian process solution."));
+      Parameters.Add(new LookupParameter<IGaussianProcessModel>(ModelParameterName, "The Gaussian process regression model to use for the solution."));
+      Parameters.Add(new LookupParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run"));
+
+      // in & out
+      Parameters.Add(new LookupParameter<ResultCollection>(ResultsParameterName, "The result collection of the algorithm."));
+      // out
+      Parameters.Add(new LookupParameter<IGaussianProcessSolution>(SolutionParameterName, "The produced Gaussian process solution."));
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
+      if (!Parameters.ContainsKey(CreateSolutionParameterName)) {
+        Parameters.Add(new LookupParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run"));
+      }
+      #endregion
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessRegressionSolutionCreator(this, cloner);
+    }
+
+    public override IOperation Apply() {
+      if (ModelParameter.ActualValue != null && CreateSolutionParameter.ActualValue.Value == true) {
+        var m = (IGaussianProcessModel)ModelParameter.ActualValue.Clone();
+        m.FixParameters();
+        var data = (IRegressionProblemData)ProblemDataParameter.ActualValue.Clone();
+        var s = new GaussianProcessRegressionSolution(m, data);
+
+
+        SolutionParameter.ActualValue = s;
+        var results = ResultsParameter.ActualValue;
+        if (!results.ContainsKey(SolutionParameterName)) {
+          results.Add(new Result(SolutionParameterName, "The Gaussian process regression solution", s));
+          results.Add(new Result(TrainingRSquaredResultName,
+                                 "The Pearson's R² of the Gaussian process solution on the training partition.",
+                                 new DoubleValue(s.TrainingRSquared)));
+          results.Add(new Result(TestRSquaredResultName,
+                                 "The Pearson's R² of the Gaussian process solution on the test partition.",
+                                 new DoubleValue(s.TestRSquared)));
+          results.Add(new Result(NegLogPseudoLikelihood,
+                                 "The negative log pseudo-likelihood (from leave-one-out-cross-validation).", 
+                                 new DoubleValue(m.LooCvNegativeLogPseudoLikelihood)));
+        } else {
+          results[SolutionParameterName].Value = s;
+          results[TrainingRSquaredResultName].Value = new DoubleValue(s.TrainingRSquared);
+          results[TestRSquaredResultName].Value = new DoubleValue(s.TestRSquared);
+          results[NegLogPseudoLikelihood].Value = new DoubleValue(m.LooCvNegativeLogPseudoLikelihood);
+        }
+      }
+      return base.Apply();
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/ICovarianceFunction.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/ICovarianceFunction.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/ICovarianceFunction.cs	(revision 17041)
@@ -0,0 +1,44 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using HEAL.Attic;
+using HeuristicLab.Core;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+
+  public delegate double CovarianceFunctionDelegate(double[,] x, int i, int j);
+  public delegate double CrossCovarianceFunctionDelegate(double[,] x, double[,] xt, int i, int j);
+  public delegate IList<double> CovarianceGradientFunctionDelegate(double[,] x, int i, int j);
+
+  public class ParameterizedCovarianceFunction {
+    public CovarianceFunctionDelegate Covariance { get; set; }
+    public CrossCovarianceFunctionDelegate CrossCovariance { get; set; }
+    public CovarianceGradientFunctionDelegate CovarianceGradient { get; set; }
+  }
+
+  [StorableType("d175f852-bb94-4b19-9afa-a7f554845a26")]
+  public interface ICovarianceFunction : IItem {
+    int GetNumberOfParameters(int numberOfVariables);
+    void SetParameter(double[] p);
+    ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices);
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/IMeanFunction.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/IMeanFunction.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/IMeanFunction.cs	(revision 17041)
@@ -0,0 +1,40 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public delegate double MeanFunctionDelegate(double[,] x, int row);
+  public delegate double MeanGradientDelegate(double[,] x, int row, int k);
+
+  public class ParameterizedMeanFunction {
+    public MeanFunctionDelegate Mean { get; set; }
+    public MeanGradientDelegate Gradient { get; set; }
+  }
+
+  [StorableType("5a0e9004-0e77-42c2-90f9-744896c1224f")]
+  public interface IMeanFunction : IItem {
+    int GetNumberOfParameters(int numberOfVariables);
+    void SetParameter(double[] p);
+    ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, int[] columnIndices);
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanConst.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanConst.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanConst.cs	(revision 17041)
@@ -0,0 +1,88 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("6E29FC23-D11B-4F32-9101-DB2BF5B2F29E")]
+  [Item(Name = "MeanConst", Description = "Constant mean function for Gaussian processes.")]
+  public sealed class MeanConst : ParameterizedNamedItem, IMeanFunction {
+    public IValueParameter<DoubleValue> ValueParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Value"]; }
+    }
+
+    [StorableConstructor]
+    private MeanConst(StorableConstructorFlag _) : base(_) { }
+    private MeanConst(MeanConst original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public MeanConst()
+      : base() {
+      this.name = ItemName;
+      this.description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Value", "The constant value for the constant mean function."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanConst(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return ValueParameter.Value != null ? 0 : 1;
+    }
+
+    public void SetParameter(double[] p) {
+      double c;
+      GetParameters(p, out c);
+      ValueParameter.Value = new DoubleValue(c);
+    }
+
+    private void GetParameters(double[] p, out double c) {
+      if (ValueParameter.Value == null) {
+        c = p[0];
+      } else {
+        if (p.Length > 0)
+          throw new ArgumentException(
+            "The length of the parameter vector does not match the number of free parameters for the constant mean function.",
+            "p");
+        c = ValueParameter.Value.Value;
+      }
+    }
+
+    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, int[] columnIndices) {
+      double c;
+      GetParameters(p, out c);
+      var mf = new ParameterizedMeanFunction();
+      mf.Mean = (x, i) => c;
+      mf.Gradient = (x, i, k) => {
+        if (k > 0) throw new ArgumentException();
+        return 1.0;
+      };
+      return mf;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanLinear.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanLinear.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanLinear.cs	(revision 17041)
@@ -0,0 +1,88 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("B5F37A3E-1004-4C24-ADDE-C830757A9ABE")]
+  [Item(Name = "MeanLinear", Description = "Linear mean function for Gaussian processes.")]
+  public sealed class MeanLinear : ParameterizedNamedItem, IMeanFunction {
+    public IValueParameter<DoubleArray> WeightsParameter {
+      get { return (IValueParameter<DoubleArray>)Parameters["Weights"]; }
+    }
+
+    [StorableConstructor]
+    private MeanLinear(StorableConstructorFlag _) : base(_) { }
+    private MeanLinear(MeanLinear original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public MeanLinear()
+      : base() {
+      Parameters.Add(new OptionalValueParameter<DoubleArray>("Weights", "The weights parameter for the linear mean function."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanLinear(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return WeightsParameter.Value != null ? 0 : numberOfVariables;
+    }
+
+    public void SetParameter(double[] p) {
+      double[] weights;
+      GetParameter(p, out weights);
+      WeightsParameter.Value = new DoubleArray(weights);
+    }
+
+    public void GetParameter(double[] p, out double[] weights) {
+      if (WeightsParameter.Value == null) {
+        weights = p;
+      } else {
+        if (p.Length != 0) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for the linear mean function.", "p");
+        weights = WeightsParameter.Value.ToArray();
+      }
+    }
+
+    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, int[] columnIndices) {
+      double[] weights;
+      int[] columns = columnIndices;
+      GetParameter(p, out weights);
+      var mf = new ParameterizedMeanFunction();
+      mf.Mean = (x, i) => {
+        // sanity check
+        if (weights.Length != columns.Length) throw new ArgumentException("The number of rparameters must match the number of variables for the linear mean function.");
+        return Util.ScalarProd(weights, Util.GetRow(x, i, columns).ToArray());
+      };
+      mf.Gradient = (x, i, k) => {
+        if (k > columns.Length) throw new ArgumentException();
+        return x[i, columns[k]];
+      };
+      return mf;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanModel.cs	(revision 17041)
@@ -0,0 +1,98 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("4A05BC94-CC08-45E0-989A-516B1C1CC956")]
+  [Item(Name = "MeanModel", Description = "A mean function for Gaussian processes that uses a regression solution created with a different algorithm to calculate the mean.")]
+  // essentially an adaptor which maps from IMeanFunction to IRegressionSolution
+  public sealed class MeanModel : ParameterizedNamedItem, IMeanFunction {
+    public IValueParameter<IRegressionSolution> RegressionSolutionParameter {
+      get { return (IValueParameter<IRegressionSolution>)Parameters["RegressionSolution"]; }
+    }
+    public IRegressionSolution RegressionSolution {
+      get { return RegressionSolutionParameter.Value; }
+      set { RegressionSolutionParameter.Value = value; }
+    }
+
+    [StorableConstructor]
+    private MeanModel(StorableConstructorFlag _) : base(_) { }
+    private MeanModel(MeanModel original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public MeanModel()
+      : base("MeanModel", "A mean function for Gaussian processes that uses a regression solution created with a different algorithm to calculate the mean.") {
+      Parameters.Add(new ValueParameter<IRegressionSolution>("RegressionSolution", "The solution containing the model that should be used for the mean prediction."));
+    }
+
+    public MeanModel(IRegressionSolution solution)
+      : this() {
+      // here we cannot check if the model is actually compatible (uses only input variables that are available)
+      // we only assume that the list of allowed inputs in the regression solution is the same as the list of allowed
+      // inputs in the Gaussian process.
+      // later we might get an error or bad behaviour when the mean function is evaluated
+      RegressionSolution = solution;
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanModel(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return 0; // no support for hyperparameters for regression models yet
+    }
+
+    public void SetParameter(double[] p) {
+      if (p.Length > 0) throw new ArgumentException("No parameters allowed for model-based mean function.", "p");
+    }
+
+    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, int[] columnIndices) {
+      if (p.Length > 0) throw new ArgumentException("No parameters allowed for model-based mean function.", "p");
+      var solution = RegressionSolution;
+      var variableNames = solution.ProblemData.AllowedInputVariables.ToArray();
+      if (variableNames.Length != columnIndices.Length)
+        throw new ArgumentException("The number of input variables does not match in MeanModel");
+      var variableValues = variableNames.Select(_ => new List<double>() { 0.0 }).ToArray(); // or of zeros
+      // uses modifyable dataset to pass values to the model
+      var ds = new ModifiableDataset(variableNames, variableValues);
+      var mf = new ParameterizedMeanFunction();
+      var model = solution.Model; // effort for parameter access only once
+      mf.Mean = (x, i) => {
+        ds.ReplaceRow(0, Util.GetRow(x, i, columnIndices).OfType<object>());
+        return model.GetEstimatedValues(ds, 0.ToEnumerable()).Single(); // evaluate the model on the specified row only
+      };
+      mf.Gradient = (x, i, k) => {
+        if (k > 0)
+          throw new ArgumentException();
+        return 0.0;
+      };
+      return mf;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanProduct.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanProduct.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanProduct.cs	(revision 17041)
@@ -0,0 +1,112 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("27D33B4E-6100-419E-B4EA-6D5EFDBFF823")]
+  [Item(Name = "MeanProduct", Description = "Product of mean functions for Gaussian processes.")]
+  public sealed class MeanProduct : Item, IMeanFunction {
+    [Storable]
+    private ItemList<IMeanFunction> factors;
+
+    [Storable]
+    private int numberOfVariables;
+
+    public ItemList<IMeanFunction> Factors {
+      get { return factors; }
+    }
+
+    [StorableConstructor]
+    private MeanProduct(StorableConstructorFlag _) : base(_) {
+    }
+
+    private MeanProduct(MeanProduct original, Cloner cloner)
+      : base(original, cloner) {
+      this.factors = cloner.Clone(original.factors);
+      this.numberOfVariables = original.numberOfVariables;
+    }
+
+    public MeanProduct() {
+      this.factors = new ItemList<IMeanFunction>();
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanProduct(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      this.numberOfVariables = numberOfVariables;
+      return factors.Select(t => t.GetNumberOfParameters(numberOfVariables)).Sum();
+    }
+
+    public void SetParameter(double[] p) {
+      int offset = 0;
+      foreach (var t in factors) {
+        var numberOfParameters = t.GetNumberOfParameters(numberOfVariables);
+        t.SetParameter(p.Skip(offset).Take(numberOfParameters).ToArray());
+        offset += numberOfParameters;
+      }
+    }
+
+
+    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, int[] columnIndices) {
+      var factorMf = new List<ParameterizedMeanFunction>();
+      int totalNumberOfParameters = GetNumberOfParameters(numberOfVariables);
+      int[] factorIndexMap = new int[totalNumberOfParameters]; // maps k-th hyperparameter to the correct mean-term
+      int[] hyperParameterIndexMap = new int[totalNumberOfParameters]; // maps k-th hyperparameter to the l-th hyperparameter of the correct mean-term
+      int c = 0;
+      // get the parameterized mean function for each term
+      for (int factorIndex = 0; factorIndex < factors.Count; factorIndex++) {
+        var numberOfParameters = factors[factorIndex].GetNumberOfParameters(numberOfVariables);
+        factorMf.Add(factors[factorIndex].GetParameterizedMeanFunction(p.Take(numberOfParameters).ToArray(), columnIndices));
+        p = p.Skip(numberOfParameters).ToArray();
+
+        for (int hyperParameterIndex = 0; hyperParameterIndex < numberOfParameters; hyperParameterIndex++) {
+          factorIndexMap[c] = factorIndex;
+          hyperParameterIndexMap[c] = hyperParameterIndex;
+          c++;
+        }
+      }
+
+      var mf = new ParameterizedMeanFunction();
+      mf.Mean = (x, i) => factorMf.Select(t => t.Mean(x, i)).Aggregate((a, b) => a * b);
+      mf.Gradient = (x, i, k) => {
+        double result = 1.0;
+        int hyperParameterFactorIndex = factorIndexMap[k];
+        for (int factorIndex = 0; factorIndex < factors.Count; factorIndex++) {
+          if (factorIndex == hyperParameterFactorIndex) {
+            // multiply gradient
+            result *= factorMf[factorIndex].Gradient(x, i, hyperParameterIndexMap[k]);
+          } else {
+            // multiply mean
+            result *= factorMf[factorIndex].Mean(x, i);
+          }
+        }
+        return result;
+      };
+      return mf;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanSum.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanSum.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanSum.cs	(revision 17041)
@@ -0,0 +1,97 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("E8BB7879-43C7-4B9A-B9AC-C9CC482BC0D7")]
+  [Item(Name = "MeanSum", Description = "Sum of mean functions for Gaussian processes.")]
+  public sealed class MeanSum : Item, IMeanFunction {
+    [Storable]
+    private ItemList<IMeanFunction> terms;
+
+    [Storable]
+    private int numberOfVariables;
+    public ItemList<IMeanFunction> Terms {
+      get { return terms; }
+    }
+
+    [StorableConstructor]
+    private MeanSum(StorableConstructorFlag _) : base(_) { }
+    private MeanSum(MeanSum original, Cloner cloner)
+      : base(original, cloner) {
+      this.terms = cloner.Clone(original.terms);
+      this.numberOfVariables = original.numberOfVariables;
+    }
+    public MeanSum() {
+      this.terms = new ItemList<IMeanFunction>();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanSum(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      this.numberOfVariables = numberOfVariables;
+      return terms.Select(t => t.GetNumberOfParameters(numberOfVariables)).Sum();
+    }
+
+    public void SetParameter(double[] p) {
+      int offset = 0;
+      foreach (var t in terms) {
+        var numberOfParameters = t.GetNumberOfParameters(numberOfVariables);
+        t.SetParameter(p.Skip(offset).Take(numberOfParameters).ToArray());
+        offset += numberOfParameters;
+      }
+    }
+
+    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, int[] columnIndices) {
+      var termMf = new List<ParameterizedMeanFunction>();
+      int totalNumberOfParameters = GetNumberOfParameters(numberOfVariables);
+      int[] termIndexMap = new int[totalNumberOfParameters]; // maps k-th parameter to the correct mean-term
+      int[] hyperParameterIndexMap = new int[totalNumberOfParameters]; // maps k-th parameter to the l-th parameter of the correct mean-term
+      int c = 0;
+      // get the parameterized mean function for each term
+      for (int termIndex = 0; termIndex < terms.Count; termIndex++) {
+        var numberOfParameters = terms[termIndex].GetNumberOfParameters(numberOfVariables);
+        termMf.Add(terms[termIndex].GetParameterizedMeanFunction(p.Take(numberOfParameters).ToArray(), columnIndices));
+        p = p.Skip(numberOfParameters).ToArray();
+
+        for (int hyperParameterIndex = 0; hyperParameterIndex < numberOfParameters; hyperParameterIndex++) {
+          termIndexMap[c] = termIndex;
+          hyperParameterIndexMap[c] = hyperParameterIndex;
+          c++;
+        }
+      }
+
+      var mf = new ParameterizedMeanFunction();
+      mf.Mean = (x, i) => termMf.Select(t => t.Mean(x, i)).Sum();
+      mf.Gradient = (x, i, k) => {
+        return termMf[termIndexMap[k]].Gradient(x, i, hyperParameterIndexMap[k]);
+      };
+      return mf;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanZero.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanZero.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanZero.cs	(revision 17041)
@@ -0,0 +1,62 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+using System;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("7B76ECDD-A7B1-450F-B542-D25E19480FC5")]
+  [Item(Name = "MeanZero", Description = "Constant zero mean function for Gaussian processes.")]
+  public sealed class MeanZero : Item, IMeanFunction {
+    [StorableConstructor]
+    private MeanZero(StorableConstructorFlag _) : base(_) { }
+    private MeanZero(MeanZero original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public MeanZero() {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanZero(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return 0;
+    }
+
+    public void SetParameter(double[] p) {
+      if (p.Length > 0) throw new ArgumentException("No parameters allowed for zero mean function.", "p");
+    }
+
+    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, int[] columnIndices) {
+      if (p.Length > 0) throw new ArgumentException("No parameters allowed for zero mean function.", "p");
+      var mf = new ParameterizedMeanFunction();
+      mf.Mean = (x, i) => 0.0;
+      mf.Gradient = (x, i, k) => {
+        if (k > 0)
+          throw new ArgumentException();
+        return 0.0;
+      };
+      return mf;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/Util.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/Util.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/Util.cs	(revision 17041)
@@ -0,0 +1,120 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  internal static class Util {
+    public static double ScalarProd(double[] v, double[] u) {
+      if (v.Length != u.Length) throw new InvalidOperationException();
+      double prod = 0.0;
+      for (int i = 0; i < v.Length; i++)
+        prod += v[i] * u[i];
+      return prod;
+    }
+
+    public static double SqrDist(IEnumerable<double> x, IEnumerable<double> y) {
+      return x.Zip(y, (a, b) => (a - b) * (a - b)).Sum();
+    }
+
+    public static double SqrDist(double x, double y) {
+      double d = x - y;
+      return d * d;
+    }
+
+    public static double SqrDist(double[,] x, int i, int j, int[] columnIndices, double scale = 1.0) {
+      return SqrDist(x, i, x, j, columnIndices, scale);
+    }
+
+    public static double SqrDist(double[,] x, int i, double[,] xt, int j, int[] columnIndices, double scale = 1.0) {
+      double ss = 0.0;
+      if (columnIndices == null) columnIndices = Enumerable.Range(0, x.GetLength(1)).ToArray();
+      for (int c = 0; c < columnIndices.Length; c++) {
+        var columnIndex = columnIndices[c];
+        double d = x[i, columnIndex] - xt[j, columnIndex];
+        ss += d * d;
+      }
+      return scale * scale * ss;
+    }
+
+    public static double SqrDist(double[,] x, int i, int j, double[] scale, int[] columnIndices) {
+      return SqrDist(x, i, x, j, scale, columnIndices);
+    }
+
+    public static double SqrDist(double[,] x, int i, double[,] xt, int j, double[] scale, int[] columnIndices) {
+      double ss = 0.0;
+      int scaleIndex = 0;
+      for (int c = 0; c < columnIndices.Length; c++) {
+        var columnIndex = columnIndices[c];
+        double d = x[i, columnIndex] - xt[j, columnIndex];
+        ss += d * d * scale[scaleIndex] * scale[scaleIndex];
+        scaleIndex++;
+      }
+      // must be at the end of scale after iterating over columnIndices
+      if (scaleIndex != scale.Length)
+        throw new ArgumentException("Lengths of scales and covariance functions does not match.");
+      return ss;
+    }
+    public static double ScalarProd(double[,] x, int i, int j, int[] columnIndices, double scale = 1.0) {
+      return ScalarProd(x, i, x, j, columnIndices, scale);
+    }
+
+    public static double ScalarProd(double[,] x, int i, double[,] xt, int j, int[] columnIndices, double scale = 1.0) {
+      double sum = 0.0;
+      for (int c = 0; c < columnIndices.Length; c++) {
+        var columnIndex = columnIndices[c];
+        sum += x[i, columnIndex] * xt[j, columnIndex];
+      }
+      return scale * scale * sum;
+    }
+    public static double ScalarProd(double[,] x, int i, int j, double[] scale, int[] columnIndices) {
+      return ScalarProd(x, i, x, j, scale, columnIndices);
+    }
+
+    public static double ScalarProd(double[,] x, int i, double[,] xt, int j, double[] scale, int[] columnIndices) {
+      double sum = 0.0;
+      int scaleIndex = 0;
+      for (int c = 0; c < columnIndices.Length; c++, scaleIndex++) {
+        var columnIndex = columnIndices[c];
+        sum += x[i, columnIndex] * scale[scaleIndex] * xt[j, columnIndex] * scale[scaleIndex];
+      }
+      // must be at the end of scale after iterating over columnIndices
+      if (scaleIndex != scale.Length)
+        throw new ArgumentException("Lengths of scales and covariance functions does not match.");
+
+      return sum;
+    }
+
+    public static IEnumerable<double> GetRow(double[,] x, int r) {
+      int cols = x.GetLength(1);
+      return GetRow(x, r, Enumerable.Range(0, cols));
+    }
+    public static IEnumerable<double> GetRow(double[,] x, int r, IEnumerable<int> columnIndices) {
+      return columnIndices.Select(c => x[r, c]);
+    }
+    public static IEnumerable<double> GetCol(double[,] x, int c) {
+      int rows = x.GetLength(0);
+      return Enumerable.Range(0, rows).Select(r => x[r, c]);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithm.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithm.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithm.cs	(revision 17041)
@@ -0,0 +1,301 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Algorithms.DataAnalysis.GradientBoostedTrees;
+using HeuristicLab.Analysis;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.PluginInfrastructure;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("Gradient Boosted Trees (GBT)", "Gradient boosted trees algorithm. Specific implementation of gradient boosting for regression trees. Friedman, J. \"Greedy Function Approximation: A Gradient Boosting Machine\", IMS 1999 Reitz Lecture.")]
+  [StorableType("8CCB55BD-4935-4868-855F-D3E5D55127AA")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 125)]
+  public class GradientBoostedTreesAlgorithm : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+    #region ParameterNames
+    private const string IterationsParameterName = "Iterations";
+    private const string MaxSizeParameterName = "Maximum Tree Size";
+    private const string NuParameterName = "Nu";
+    private const string RParameterName = "R";
+    private const string MParameterName = "M";
+    private const string SeedParameterName = "Seed";
+    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+    private const string LossFunctionParameterName = "LossFunction";
+    private const string UpdateIntervalParameterName = "UpdateInterval";
+    private const string ModelCreationParameterName = "ModelCreation";
+    #endregion
+
+    #region ParameterProperties
+    public IFixedValueParameter<IntValue> IterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[IterationsParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> MaxSizeParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[MaxSizeParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> NuParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[NuParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> RParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[RParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> MParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[MParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> SeedParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+    public FixedValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (FixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+    public IConstrainedValueParameter<ILossFunction> LossFunctionParameter {
+      get { return (IConstrainedValueParameter<ILossFunction>)Parameters[LossFunctionParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> UpdateIntervalParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[UpdateIntervalParameterName]; }
+    }
+    private IFixedValueParameter<EnumValue<ModelCreation>> ModelCreationParameter {
+      get { return (IFixedValueParameter<EnumValue<ModelCreation>>)Parameters[ModelCreationParameterName]; }
+    }
+    #endregion
+
+    #region Properties
+    public int Iterations {
+      get { return IterationsParameter.Value.Value; }
+      set { IterationsParameter.Value.Value = value; }
+    }
+    public int Seed {
+      get { return SeedParameter.Value.Value; }
+      set { SeedParameter.Value.Value = value; }
+    }
+    public bool SetSeedRandomly {
+      get { return SetSeedRandomlyParameter.Value.Value; }
+      set { SetSeedRandomlyParameter.Value.Value = value; }
+    }
+    public int MaxSize {
+      get { return MaxSizeParameter.Value.Value; }
+      set { MaxSizeParameter.Value.Value = value; }
+    }
+    public double Nu {
+      get { return NuParameter.Value.Value; }
+      set { NuParameter.Value.Value = value; }
+    }
+    public double R {
+      get { return RParameter.Value.Value; }
+      set { RParameter.Value.Value = value; }
+    }
+    public double M {
+      get { return MParameter.Value.Value; }
+      set { MParameter.Value.Value = value; }
+    }
+    public ModelCreation ModelCreation {
+      get { return ModelCreationParameter.Value.Value; }
+      set { ModelCreationParameter.Value.Value = value; }
+    }
+    #endregion
+
+    #region ResultsProperties
+    private double ResultsBestQuality {
+      get { return ((DoubleValue)Results["Best Quality"].Value).Value; }
+      set { ((DoubleValue)Results["Best Quality"].Value).Value = value; }
+    }
+    private DataTable ResultsQualities {
+      get { return ((DataTable)Results["Qualities"].Value); }
+    }
+    #endregion
+
+    [StorableConstructor]
+    protected GradientBoostedTreesAlgorithm(StorableConstructorFlag _) : base(_) { }
+
+    protected GradientBoostedTreesAlgorithm(GradientBoostedTreesAlgorithm original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GradientBoostedTreesAlgorithm(this, cloner);
+    }
+
+    public GradientBoostedTreesAlgorithm() {
+      Problem = new RegressionProblem(); // default problem
+
+      Parameters.Add(new FixedValueParameter<IntValue>(IterationsParameterName, "Number of iterations (set as high as possible, adjust in combination with nu, when increasing iterations also decrease nu)", new IntValue(1000)));
+      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<IntValue>(MaxSizeParameterName, "Maximal size of the tree learned in each step (prefer smaller sizes (3 to 10) if possible)", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(RParameterName, "Ratio of training rows selected randomly in each step (0 < R <= 1)", new DoubleValue(0.5)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName, "Ratio of variables selected randomly in each step (0 < M <= 1)", new DoubleValue(0.5)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(NuParameterName, "Learning rate nu (step size for the gradient update, should be small 0 < nu < 0.1)", new DoubleValue(0.002)));
+      Parameters.Add(new FixedValueParameter<IntValue>(UpdateIntervalParameterName, "", new IntValue(100)));
+      Parameters[UpdateIntervalParameterName].Hidden = true;
+      Parameters.Add(new FixedValueParameter<EnumValue<ModelCreation>>(ModelCreationParameterName, "Defines the results produced at the end of the run (Surrogate => Less disk space, lazy recalculation of model)", new EnumValue<ModelCreation>(ModelCreation.Model)));
+      Parameters[ModelCreationParameterName].Hidden = true;
+
+      var lossFunctions = ApplicationManager.Manager.GetInstances<ILossFunction>();
+      Parameters.Add(new ConstrainedValueParameter<ILossFunction>(LossFunctionParameterName, "The loss function", new ItemSet<ILossFunction>(lossFunctions)));
+      LossFunctionParameter.Value = LossFunctionParameter.ValidValues.First(f => f.ToString().Contains("Squared")); // squared error loss is the default
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.4
+      #region Backwards compatible code, remove with 3.5
+
+      #region LossFunction
+      // parameter type has been changed
+      var lossFunctionParam = Parameters[LossFunctionParameterName] as ConstrainedValueParameter<StringValue>;
+      if (lossFunctionParam != null) {
+        Parameters.Remove(LossFunctionParameterName);
+        var selectedValue = lossFunctionParam.Value; // to be restored below
+
+        var lossFunctions = ApplicationManager.Manager.GetInstances<ILossFunction>();
+        Parameters.Add(new ConstrainedValueParameter<ILossFunction>(LossFunctionParameterName, "The loss function", new ItemSet<ILossFunction>(lossFunctions)));
+        // try to restore selected value
+        var selectedLossFunction =
+          LossFunctionParameter.ValidValues.FirstOrDefault(f => f.ToString() == selectedValue.Value);
+        if (selectedLossFunction != null) {
+          LossFunctionParameter.Value = selectedLossFunction;
+        } else {
+          LossFunctionParameter.Value = LossFunctionParameter.ValidValues.First(f => f.ToString().Contains("Squared")); // default: SE
+        }
+      }
+      #endregion
+
+      #region CreateSolution
+      // parameter type has been changed
+      if (Parameters.ContainsKey("CreateSolution")) {
+        var createSolutionParam = Parameters["CreateSolution"] as FixedValueParameter<BoolValue>;
+        Parameters.Remove(createSolutionParam);
+
+        ModelCreation value = createSolutionParam.Value.Value ? ModelCreation.Model : ModelCreation.QualityOnly;
+        Parameters.Add(new FixedValueParameter<EnumValue<ModelCreation>>(ModelCreationParameterName, "Defines the results produced at the end of the run (Surrogate => Less disk space, lazy recalculation of model)", new EnumValue<ModelCreation>(value)));
+        Parameters[ModelCreationParameterName].Hidden = true;
+      }
+      #endregion
+      #endregion
+    }
+
+    protected override void Run(CancellationToken cancellationToken) {
+      // Set up the algorithm
+      if (SetSeedRandomly) Seed = Random.RandomSeedGenerator.GetSeed();
+
+      // Set up the results display
+      var iterations = new IntValue(0);
+      Results.Add(new Result("Iterations", iterations));
+
+      var table = new DataTable("Qualities");
+      table.Rows.Add(new DataRow("Loss (train)"));
+      table.Rows.Add(new DataRow("Loss (test)"));
+      table.Rows["Loss (train)"].VisualProperties.StartIndexZero = true;
+      table.Rows["Loss (test)"].VisualProperties.StartIndexZero = true;
+
+      Results.Add(new Result("Qualities", table));
+      var curLoss = new DoubleValue();
+      Results.Add(new Result("Loss (train)", curLoss));
+
+      // init
+      var problemData = (IRegressionProblemData)Problem.ProblemData.Clone();
+      var lossFunction = LossFunctionParameter.Value;
+      var state = GradientBoostedTreesAlgorithmStatic.CreateGbmState(problemData, lossFunction, (uint)Seed, MaxSize, R, M, Nu);
+
+      var updateInterval = UpdateIntervalParameter.Value.Value;
+      // Loop until iteration limit reached or canceled.
+      for (int i = 0; i < Iterations; i++) {
+        cancellationToken.ThrowIfCancellationRequested();
+
+        GradientBoostedTreesAlgorithmStatic.MakeStep(state);
+
+        // iteration results
+        if (i % updateInterval == 0) {
+          curLoss.Value = state.GetTrainLoss();
+          table.Rows["Loss (train)"].Values.Add(curLoss.Value);
+          table.Rows["Loss (test)"].Values.Add(state.GetTestLoss());
+          iterations.Value = i;
+        }
+      }
+
+      // final results
+      iterations.Value = Iterations;
+      curLoss.Value = state.GetTrainLoss();
+      table.Rows["Loss (train)"].Values.Add(curLoss.Value);
+      table.Rows["Loss (test)"].Values.Add(state.GetTestLoss());
+
+      // produce variable relevance
+      var orderedImpacts = state.GetVariableRelevance().Select(t => new { name = t.Key, impact = t.Value }).ToList();
+
+      var impacts = new DoubleMatrix();
+      var matrix = impacts as IStringConvertibleMatrix;
+      matrix.Rows = orderedImpacts.Count;
+      matrix.RowNames = orderedImpacts.Select(x => x.name);
+      matrix.Columns = 1;
+      matrix.ColumnNames = new string[] { "Relative variable relevance" };
+
+      int rowIdx = 0;
+      foreach (var p in orderedImpacts) {
+        matrix.SetValue(string.Format("{0:N2}", p.impact), rowIdx++, 0);
+      }
+
+      Results.Add(new Result("Variable relevance", impacts));
+      Results.Add(new Result("Loss (test)", new DoubleValue(state.GetTestLoss())));
+
+      // produce solution 
+      if (ModelCreation == ModelCreation.SurrogateModel || ModelCreation == ModelCreation.Model) {
+        IRegressionModel model = state.GetModel();
+
+        if (ModelCreation == ModelCreation.SurrogateModel) {
+          model = new GradientBoostedTreesModelSurrogate(problemData, (uint)Seed, lossFunction, Iterations, MaxSize, R, M, Nu, (GradientBoostedTreesModel)model);
+        }
+
+        // for logistic regression we produce a classification solution
+        if (lossFunction is LogisticRegressionLoss) {
+          var classificationModel = new DiscriminantFunctionClassificationModel(model,
+            new AccuracyMaximizationThresholdCalculator());
+          var classificationProblemData = new ClassificationProblemData(problemData.Dataset,
+            problemData.AllowedInputVariables, problemData.TargetVariable, problemData.Transformations);
+          classificationProblemData.TrainingPartition.Start = Problem.ProblemData.TrainingPartition.Start;
+          classificationProblemData.TrainingPartition.End = Problem.ProblemData.TrainingPartition.End;
+          classificationProblemData.TestPartition.Start = Problem.ProblemData.TestPartition.Start;
+          classificationProblemData.TestPartition.End = Problem.ProblemData.TestPartition.End;
+
+          classificationModel.SetThresholdsAndClassValues(new double[] { double.NegativeInfinity, 0.0 }, new[] { 0.0, 1.0 });
+
+
+          var classificationSolution = new DiscriminantFunctionClassificationSolution(classificationModel, classificationProblemData);
+          Results.Add(new Result("Solution", classificationSolution));
+        } else {
+          // otherwise we produce a regression solution
+          Results.Add(new Result("Solution", new GradientBoostedTreesSolution(model, problemData)));
+        }
+      } else if (ModelCreation == ModelCreation.QualityOnly) {
+        //Do nothing
+      } else {
+        throw new NotImplementedException("Selected parameter for CreateSolution isn't implemented yet");
+      }
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithmStatic.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithmStatic.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithmStatic.cs	(revision 17041)
@@ -0,0 +1,201 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.Contracts;
+using System.Linq;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Random;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public static class GradientBoostedTreesAlgorithmStatic {
+    #region static API
+
+    public interface IGbmState {
+      IRegressionModel GetModel();
+      double GetTrainLoss();
+      double GetTestLoss();
+      IEnumerable<KeyValuePair<string, double>> GetVariableRelevance();
+    }
+
+    // created through factory method
+    // GbmState details are private API users can only use methods from IGbmState
+    private class GbmState : IGbmState {
+      internal IRegressionProblemData problemData { get; private set; }
+      internal ILossFunction lossFunction { get; private set; }
+      internal int maxSize { get; private set; }
+      internal double nu { get; private set; }
+      internal double r { get; private set; }
+      internal double m { get; private set; }
+      internal int[] trainingRows { get; private set; }
+      internal int[] testRows { get; private set; }
+      internal RegressionTreeBuilder treeBuilder { get; private set; }
+
+      private readonly uint randSeed;
+      private MersenneTwister random { get; set; }
+
+      // array members (allocate only once)
+      internal double[] pred;
+      internal double[] predTest;
+      internal double[] y;
+      internal int[] activeIdx;
+      internal double[] pseudoRes;
+
+      private readonly IList<IRegressionModel> models;
+      private readonly IList<double> weights;
+
+      public GbmState(IRegressionProblemData problemData, ILossFunction lossFunction, uint randSeed, int maxSize, double r, double m, double nu) {
+        // default settings for MaxSize, Nu and R
+        this.maxSize = maxSize;
+        this.nu = nu;
+        this.r = r;
+        this.m = m;
+
+        this.randSeed = randSeed;
+        random = new MersenneTwister(randSeed);
+        this.problemData = problemData;
+        this.trainingRows = problemData.TrainingIndices.ToArray();
+        this.testRows = problemData.TestIndices.ToArray();
+        this.lossFunction = lossFunction;
+
+        int nRows = trainingRows.Length;
+
+        y = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, trainingRows).ToArray();
+
+        treeBuilder = new RegressionTreeBuilder(problemData, random);
+
+        activeIdx = Enumerable.Range(0, nRows).ToArray();
+
+        var zeros = Enumerable.Repeat(0.0, nRows).ToArray();
+        double f0 = lossFunction.LineSearch(y, zeros, activeIdx, 0, nRows - 1); // initial constant value (mean for squared errors)
+        pred = Enumerable.Repeat(f0, nRows).ToArray();
+        predTest = Enumerable.Repeat(f0, testRows.Length).ToArray();
+        pseudoRes = new double[nRows];
+
+        models = new List<IRegressionModel>();
+        weights = new List<double>();
+        // add constant model
+        models.Add(new ConstantModel(f0, problemData.TargetVariable));
+        weights.Add(1.0);
+      }
+
+      public IRegressionModel GetModel() {
+        return new GradientBoostedTreesModel(models, weights);
+      }
+      public IEnumerable<KeyValuePair<string, double>> GetVariableRelevance() {
+        return treeBuilder.GetVariableRelevance();
+      }
+
+      public double GetTrainLoss() {
+        int nRows = y.Length;
+        return lossFunction.GetLoss(y, pred) / nRows;
+      }
+      public double GetTestLoss() {
+        var yTest = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, testRows);
+        var nRows = testRows.Length;
+        return lossFunction.GetLoss(yTest, predTest) / nRows;
+      }
+
+      internal void AddModel(IRegressionModel m, double weight) {
+        models.Add(m);
+        weights.Add(weight);
+      }
+    }
+
+    // simple interface
+    public static GradientBoostedTreesSolution TrainGbm(IRegressionProblemData problemData, ILossFunction lossFunction, int maxSize, double nu, double r, double m, int maxIterations, uint randSeed = 31415) {
+      Contract.Assert(r > 0);
+      Contract.Assert(r <= 1.0);
+      Contract.Assert(nu > 0);
+      Contract.Assert(nu <= 1.0);
+
+      var state = (GbmState)CreateGbmState(problemData, lossFunction, randSeed, maxSize, r, m, nu);
+
+      for (int iter = 0; iter < maxIterations; iter++) {
+        MakeStep(state);
+      }
+
+      var model = state.GetModel();
+      return new GradientBoostedTreesSolution(model, (IRegressionProblemData)problemData.Clone());
+    }
+
+    // for custom stepping & termination
+    public static IGbmState CreateGbmState(IRegressionProblemData problemData, ILossFunction lossFunction, uint randSeed, int maxSize = 3, double r = 0.66, double m = 0.5, double nu = 0.01) {
+      // check input variables. Only double variables are allowed.
+      var invalidInputs =
+        problemData.AllowedInputVariables.Where(name => !problemData.Dataset.VariableHasType<double>(name));
+      if (invalidInputs.Any())
+        throw new NotSupportedException("Gradient tree boosting only supports real-valued variables. Unsupported inputs: " + string.Join(", ", invalidInputs));
+
+      return new GbmState(problemData, lossFunction, randSeed, maxSize, r, m, nu);
+    }
+
+    // use default settings for maxSize, nu, r from state
+    public static void MakeStep(IGbmState state) {
+      var gbmState = state as GbmState;
+      if (gbmState == null) throw new ArgumentException("state");
+
+      MakeStep(gbmState, gbmState.maxSize, gbmState.nu, gbmState.r, gbmState.m);
+    }
+
+    // allow dynamic adaptation of maxSize, nu and r (even though this is not used)
+    public static void MakeStep(IGbmState state, int maxSize, double nu, double r, double m) {
+      var gbmState = state as GbmState;
+      if (gbmState == null) throw new ArgumentException("state");
+
+      var problemData = gbmState.problemData;
+      var lossFunction = gbmState.lossFunction;
+      var yPred = gbmState.pred;
+      var yPredTest = gbmState.predTest;
+      var treeBuilder = gbmState.treeBuilder;
+      var y = gbmState.y;
+      var activeIdx = gbmState.activeIdx;
+      var pseudoRes = gbmState.pseudoRes;
+      var trainingRows = gbmState.trainingRows;
+      var testRows = gbmState.testRows;
+
+      // copy output of gradient function to pre-allocated rim array (pseudo-residual per row and model)
+      int rimIdx = 0;
+      foreach (var g in lossFunction.GetLossGradient(y, yPred)) {
+        pseudoRes[rimIdx++] = g;
+      }
+
+      var tree = treeBuilder.CreateRegressionTreeForGradientBoosting(pseudoRes, yPred, maxSize, activeIdx, lossFunction, r, m);
+
+      int i = 0;
+      foreach (var pred in tree.GetEstimatedValues(problemData.Dataset, trainingRows)) {
+        yPred[i] = yPred[i] + nu * pred;
+        i++;
+      }
+      // update predictions for validation set
+      i = 0;
+      foreach (var pred in tree.GetEstimatedValues(problemData.Dataset, testRows)) {
+        yPredTest[i] = yPredTest[i] + nu * pred;
+        i++;
+      }
+
+      gbmState.AddModel(tree, nu);
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesModel.cs	(revision 17041)
@@ -0,0 +1,111 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("4EC1B359-D145-434C-A373-3EDD764D2D63")]
+  [Item("Gradient boosted trees model", "")]
+  // this is essentially a collection of weighted regression models
+  public sealed class GradientBoostedTreesModel : RegressionModel, IGradientBoostedTreesModel {
+    // BackwardsCompatibility3.4 for allowing deserialization & serialization of old models
+    #region Backwards compatible code, remove with 3.5
+
+    [Storable(Name = "models")]
+    private IList<IRegressionModel> __persistedModels {
+      set {
+        this.models.Clear();
+        foreach (var m in value) this.models.Add(m);
+      }
+      get { return models; }
+    }
+    [Storable(Name = "weights")]
+    private IList<double> __persistedWeights {
+      set {
+        this.weights.Clear();
+        foreach (var w in value) this.weights.Add(w);
+      }
+      get { return weights; }
+    }
+    #endregion
+
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return models.SelectMany(x => x.VariablesUsedForPrediction).Distinct().OrderBy(x => x); }
+    }
+
+    private readonly IList<IRegressionModel> models;
+    public IEnumerable<IRegressionModel> Models { get { return models; } }
+
+    private readonly IList<double> weights;
+    public IEnumerable<double> Weights { get { return weights; } }
+
+    [StorableConstructor]
+    private GradientBoostedTreesModel(StorableConstructorFlag _) : base(_) {
+      models = new List<IRegressionModel>();
+      weights = new List<double>();
+    }
+    private GradientBoostedTreesModel(GradientBoostedTreesModel original, Cloner cloner)
+      : base(original, cloner) {
+      this.weights = new List<double>(original.weights);
+      this.models = new List<IRegressionModel>(original.models.Select(m => cloner.Clone(m)));
+    }
+
+    internal GradientBoostedTreesModel(IEnumerable<IRegressionModel> models, IEnumerable<double> weights)
+      : base(string.Empty, "Gradient boosted tree model", string.Empty) {
+      this.models = new List<IRegressionModel>(models);
+      this.weights = new List<double>(weights);
+
+      if (this.models.Count != this.weights.Count) throw new ArgumentException();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GradientBoostedTreesModel(this, cloner);
+    }
+
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      // allocate target array go over all models and add up weighted estimation for each row
+      if (!rows.Any()) return Enumerable.Empty<double>(); // return immediately if rows is empty. This prevents multiple iteration over lazy rows enumerable.
+      // (which essentially looks up indexes in a dictionary)
+      var res = new double[rows.Count()];
+      for (int i = 0; i < models.Count; i++) {
+        var w = weights[i];
+        var m = models[i];
+        int r = 0;
+        foreach (var est in m.GetEstimatedValues(dataset, rows)) {
+          res[r++] += w * est;
+        }
+      }
+      return res;
+    }
+
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RegressionSolution(this, (IRegressionProblemData)problemData.Clone());
+    }
+
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesModelSurrogate.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesModelSurrogate.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesModelSurrogate.cs	(revision 17041)
@@ -0,0 +1,147 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("1BF7BEFB-6739-48AA-89BC-B632E72D148C")]
+  // this class is used as a surrogate for persistence of an actual GBT model 
+  // since the actual GBT model would be very large when persisted we only store all necessary information to
+  // recalculate the actual GBT model on demand
+  [Item("Gradient boosted tree model", "")]
+  public sealed class GradientBoostedTreesModelSurrogate : RegressionModel, IGradientBoostedTreesModel {
+    // don't store the actual model!
+    // the actual model is only recalculated when necessary
+    private readonly Lazy<IGradientBoostedTreesModel> actualModel;
+    private IGradientBoostedTreesModel ActualModel {
+      get { return actualModel.Value; }
+    }
+
+    [Storable]
+    private readonly IRegressionProblemData trainingProblemData;
+    [Storable]
+    private readonly uint seed;
+    [Storable]
+    private ILossFunction lossFunction;
+    [Storable]
+    private double r;
+    [Storable]
+    private double m;
+    [Storable]
+    private double nu;
+    [Storable]
+    private int iterations;
+    [Storable]
+    private int maxSize;
+
+
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get {
+        return ActualModel.Models.SelectMany(x => x.VariablesUsedForPrediction).Distinct().OrderBy(x => x);
+      }
+    }
+
+    [StorableConstructor]
+    private GradientBoostedTreesModelSurrogate(StorableConstructorFlag _) : base(_) {
+      actualModel = new Lazy<IGradientBoostedTreesModel>(() => RecalculateModel());
+    }
+
+    private GradientBoostedTreesModelSurrogate(GradientBoostedTreesModelSurrogate original, Cloner cloner)
+      : base(original, cloner) {
+      IGradientBoostedTreesModel clonedModel = null;
+      if (original.ActualModel != null) clonedModel = cloner.Clone(original.ActualModel);
+      actualModel = new Lazy<IGradientBoostedTreesModel>(CreateLazyInitFunc(clonedModel)); // only capture clonedModel in the closure
+
+      this.trainingProblemData = cloner.Clone(original.trainingProblemData);
+      this.lossFunction = cloner.Clone(original.lossFunction);
+      this.seed = original.seed;
+      this.iterations = original.iterations;
+      this.maxSize = original.maxSize;
+      this.r = original.r;
+      this.m = original.m;
+      this.nu = original.nu;
+    }
+
+    private Func<IGradientBoostedTreesModel> CreateLazyInitFunc(IGradientBoostedTreesModel clonedModel) {
+      return () => {
+        return clonedModel == null ? RecalculateModel() : clonedModel;
+      };
+    }
+
+    // create only the surrogate model without an actual model
+    public GradientBoostedTreesModelSurrogate(IRegressionProblemData trainingProblemData, uint seed,
+      ILossFunction lossFunction, int iterations, int maxSize, double r, double m, double nu)
+      : base(trainingProblemData.TargetVariable, "Gradient boosted tree model", string.Empty) {
+      this.trainingProblemData = trainingProblemData;
+      this.seed = seed;
+      this.lossFunction = lossFunction;
+      this.iterations = iterations;
+      this.maxSize = maxSize;
+      this.r = r;
+      this.m = m;
+      this.nu = nu;
+    }
+
+    // wrap an actual model in a surrograte
+    public GradientBoostedTreesModelSurrogate(IRegressionProblemData trainingProblemData, uint seed,
+      ILossFunction lossFunction, int iterations, int maxSize, double r, double m, double nu,
+      IGradientBoostedTreesModel model)
+      : this(trainingProblemData, seed, lossFunction, iterations, maxSize, r, m, nu) {
+      actualModel = new Lazy<IGradientBoostedTreesModel>(() => model);
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GradientBoostedTreesModelSurrogate(this, cloner);
+    }
+
+    // forward message to actual model (recalculate model first if necessary)
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      return ActualModel.GetEstimatedValues(dataset, rows);
+    }
+
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RegressionSolution(this, (IRegressionProblemData)problemData.Clone());
+    }
+
+    private IGradientBoostedTreesModel RecalculateModel() {
+      return GradientBoostedTreesAlgorithmStatic.TrainGbm(trainingProblemData, lossFunction, maxSize, nu, r, m, iterations, seed).Model;
+    }
+
+    public IEnumerable<IRegressionModel> Models {
+      get {
+        return ActualModel.Models;
+      }
+    }
+
+    public IEnumerable<double> Weights {
+      get {
+        return ActualModel.Weights;
+      }
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesSolution.cs	(revision 17041)
@@ -0,0 +1,48 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("B05C382A-A637-4CD6-8BCC-E1E92B60F330")]
+  public sealed class GradientBoostedTreesSolution : RegressionSolution {
+    public new IGradientBoostedTreesModel Model {
+      get { return (IGradientBoostedTreesModel)base.Model; }
+    }
+
+
+    [StorableConstructor]
+    private GradientBoostedTreesSolution(StorableConstructorFlag _) : base(_) {
+    }
+    private GradientBoostedTreesSolution(GradientBoostedTreesSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public GradientBoostedTreesSolution(IRegressionModel model, IRegressionProblemData problemData)
+      : base(model, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GradientBoostedTreesSolution(this, cloner);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/IGradientBoostedTreesModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/IGradientBoostedTreesModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/IGradientBoostedTreesModel.cs	(revision 17041)
@@ -0,0 +1,33 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("e6c22779-7059-416d-8804-0022406b7136")]
+  public interface IGradientBoostedTreesModel : IRegressionModel {
+    IEnumerable<IRegressionModel> Models { get; }
+    IEnumerable<double> Weights { get; }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/AbsoluteErrorLoss.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/AbsoluteErrorLoss.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/AbsoluteErrorLoss.cs	(revision 17041)
@@ -0,0 +1,94 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  // loss function for the weighted absolute error
+  [StorableType("C4429BDA-665F-48A2-B18F-EA1569083842")]
+  [Item("Absolute error loss", "")]
+  public sealed class AbsoluteErrorLoss : Item, ILossFunction {
+    public AbsoluteErrorLoss() { }
+
+    public double GetLoss(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+
+      double s = 0;
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        double res = targetEnum.Current - predEnum.Current;
+        s += Math.Abs(res);  // |res|
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have differing lengths");
+
+      return s;
+    }
+
+    public IEnumerable<double> GetLossGradient(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        // dL(y, f(x)) / df(x) = sign(res)
+        var res = targetEnum.Current - predEnum.Current;
+        if (res > 0) yield return 1.0;
+        else if (res < 0) yield return -1.0;
+        else yield return 0.0;
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have differing lengths");
+    }
+
+    // return median of residuals
+    // targetArr and predArr are not changed by LineSearch
+    public double LineSearch(double[] targetArr, double[] predArr, int[] idx, int startIdx, int endIdx) {
+      if (targetArr.Length != predArr.Length)
+        throw new ArgumentException("target and pred have differing lengths");
+
+      // Median() is allocating an array anyway
+      // It would be possible to pre-allocated an array for the residuals if Median() would allow specification of a sub-range
+      int nRows = endIdx - startIdx + 1;
+      var res = new double[nRows];
+      for (int i = startIdx; i <= endIdx; i++) {
+        var row = idx[i];
+        res[i - startIdx] = targetArr[row] - predArr[row];
+      }
+      return res.Median(); // TODO: improve efficiency
+    }
+
+    #region item implementation
+    [StorableConstructor]
+    private AbsoluteErrorLoss(StorableConstructorFlag _) : base(_) { }
+
+    private AbsoluteErrorLoss(AbsoluteErrorLoss original, Cloner cloner) : base(original, cloner) { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new AbsoluteErrorLoss(this, cloner);
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/ILossFunction.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/ILossFunction.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/ILossFunction.cs	(revision 17041)
@@ -0,0 +1,44 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("588270d5-61ee-4906-b30f-841f64cd6724")]
+  // represents an interface for loss functions used by gradient boosting
+  // target represents the target vector  (original targets from the problem data, never changed)
+  // pred   represents the current vector of predictions (a weighted combination of models learned so far, this vector is updated after each step)
+  public interface ILossFunction : IItem {
+    // returns the loss of the current prediction vector 
+    double GetLoss(IEnumerable<double> target, IEnumerable<double> pred);
+
+    // returns an enumerable of the loss gradient for each row
+    IEnumerable<double> GetLossGradient(IEnumerable<double> target, IEnumerable<double> pred);
+
+    // returns the optimal value for the partition of rows stored in idx[startIdx] .. idx[endIdx] inclusive
+    double LineSearch(double[] targetArr, double[] predArr, int[] idx, int startIdx, int endIdx);
+  }
+}
+
+
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/LogisticRegressionLoss.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/LogisticRegressionLoss.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/LogisticRegressionLoss.cs	(revision 17041)
@@ -0,0 +1,102 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  // Greedy Function Approximation: A Gradient Boosting Machine (page 9) 
+  [StorableType("E91BD71E-9A1D-4352-BD68-062290F8BE9C")]
+  [Item("Logistic regression loss", "")]
+  public sealed class LogisticRegressionLoss : Item, ILossFunction {
+    public LogisticRegressionLoss() { }
+
+    public double GetLoss(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+
+      double s = 0;
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        Debug.Assert(targetEnum.Current.IsAlmost(0.0) || targetEnum.Current.IsAlmost(1.0), "labels must be 0 or 1 for logistic regression loss");
+
+        var y = targetEnum.Current * 2 - 1; // y in {-1,1}
+        s += Math.Log(1 + Math.Exp(-2 * y * predEnum.Current));
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have different lengths");
+
+      return s;
+    }
+
+    public IEnumerable<double> GetLossGradient(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        Debug.Assert(targetEnum.Current.IsAlmost(0.0) || targetEnum.Current.IsAlmost(1.0), "labels must be 0 or 1 for logistic regression loss");
+        var y = targetEnum.Current * 2 - 1; // y in {-1,1}
+
+        yield return 2 * y / (1 + Math.Exp(2 * y * predEnum.Current));
+
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have different lengths");
+    }
+
+    // targetArr and predArr are not changed by LineSearch
+    public double LineSearch(double[] targetArr, double[] predArr, int[] idx, int startIdx, int endIdx) {
+      if (targetArr.Length != predArr.Length)
+        throw new ArgumentException("target and pred have different lengths");
+
+      // "Simple Newton-Raphson step" of eqn. 23 
+      double sumY = 0.0;
+      double sumDiff = 0.0;
+      for (int i = startIdx; i <= endIdx; i++) {
+        var row = idx[i];
+        var y = targetArr[row] * 2 - 1; // y in {-1,1}
+        var pseudoResponse = 2 * y / (1 + Math.Exp(2 * y * predArr[row]));
+
+        sumY += pseudoResponse;
+        sumDiff += Math.Abs(pseudoResponse) * (2 - Math.Abs(pseudoResponse));
+      }
+      // prevent divByZero
+      sumDiff = Math.Max(1E-12, sumDiff);
+      return sumY / sumDiff;
+    }
+
+    #region item implementation
+    [StorableConstructor]
+    private LogisticRegressionLoss(StorableConstructorFlag _) : base(_) { }
+
+    private LogisticRegressionLoss(LogisticRegressionLoss original, Cloner cloner) : base(original, cloner) { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new LogisticRegressionLoss(this, cloner);
+    }
+    #endregion
+
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/QuantileRegressionLoss.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/QuantileRegressionLoss.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/QuantileRegressionLoss.cs	(revision 17041)
@@ -0,0 +1,118 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  // loss function for quantile regression
+  // Generalized Boosted Models - A Guide To The gbm Package, Greg Ridgeway, August 2007, page 11 
+  [StorableType("B8EF6C18-C1A8-4B43-8FEA-A6900287ADDD")]
+  [Item("QuantileRegressionloss", "Loss function for quantile regression")]
+  public sealed class QuantileRegressionLoss : ParameterizedNamedItem, ILossFunction {
+    public IFixedValueParameter<PercentValue> AlphaParameter {
+      get { return (IFixedValueParameter<PercentValue>)Parameters["Alpha"]; }
+    }
+
+    public double Alpha {
+      get { return AlphaParameter.Value.Value; }
+      set {
+        if (value <= 0.0 || value >= 1.0) throw new ArgumentException("Valid values for alpha: 0 < alpha < 1");
+        AlphaParameter.Value.Value = value;
+      }
+    }
+
+    public QuantileRegressionLoss()
+      : base("QuantileRegressionLoss", "Loss function for quantile regression") {
+      Parameters.Add(new FixedValueParameter<PercentValue>("Alpha", new PercentValue(0.9)));
+    }
+
+    public double GetLoss(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+      var alpha = Alpha;
+      double leftSum = 0;
+      double rightsum = 0;
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        double res = targetEnum.Current - predEnum.Current;
+        if (res > 0) leftSum += res;
+        else rightsum += -res;
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have differing lengths");
+
+      return alpha * leftSum + (1 - alpha) * rightsum;
+    }
+
+    public IEnumerable<double> GetLossGradient(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+      var alpha = AlphaParameter.Value.Value;
+
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        var res = targetEnum.Current - predEnum.Current;
+        if (res > 0) yield return alpha;
+        else if (res < 0) yield return -(1.0 - alpha);
+        else yield return 0.0;
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have differing lengths");
+    }
+
+    // targetArr and predArr are not changed by LineSearch
+    public double LineSearch(double[] targetArr, double[] predArr, int[] idx, int startIdx, int endIdx) {
+      if (targetArr.Length != predArr.Length)
+        throw new ArgumentException("target and pred have differing lengths");
+
+      // Quantile() is allocating an array anyway
+      // It would be possible to pre-allocated an array for the residuals if Quantile() would allow specification of a sub-range
+      int nRows = endIdx - startIdx + 1;
+      var res = new double[nRows];
+      for (int i = startIdx; i <= endIdx; i++) {
+        var row = idx[i];
+        res[i - startIdx] = targetArr[row] - predArr[row];
+      }
+      return res.Quantile(Alpha);
+    }
+
+    #region item implementation
+    [StorableConstructor]
+    private QuantileRegressionLoss(StorableConstructorFlag _) : base(_) { }
+
+    private QuantileRegressionLoss(QuantileRegressionLoss original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new QuantileRegressionLoss(this, cloner);
+    }
+    #endregion
+
+    public override bool CanChangeName { get { return false; } }
+    public override bool CanChangeDescription { get { return false; } }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/RelativeErrorLoss.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/RelativeErrorLoss.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/RelativeErrorLoss.cs	(revision 17041)
@@ -0,0 +1,122 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  // relative error loss is a special case of weighted absolute error loss with weights = (1/target)
+  [StorableType("1334143A-7F77-46C1-8080-0CC4F4DA8995")]
+  [Item("Relative error loss", "")]
+  public sealed class RelativeErrorLoss : Item, ILossFunction {
+    public RelativeErrorLoss() { }
+
+    public double GetLoss(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+
+      double s = 0;
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        double res = targetEnum.Current - predEnum.Current;
+        s += Math.Abs(res) * Math.Abs(1.0 / targetEnum.Current);
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have different lengths");
+
+      return s;
+    }
+
+    public IEnumerable<double> GetLossGradient(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        // sign(res) * abs(1 / target)
+        var res = targetEnum.Current - predEnum.Current;
+        if (res > 0) yield return 1.0 / Math.Abs(targetEnum.Current);
+        else if (res < 0) yield return -1.0 / Math.Abs(targetEnum.Current);
+        else yield return 0.0;
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have different lengths");
+    }
+
+    // targetArr and predArr are not changed by LineSearch
+    public double LineSearch(double[] targetArr, double[] predArr, int[] idx, int startIdx, int endIdx) {
+      if (targetArr.Length != predArr.Length)
+        throw new ArgumentException("target and pred have different lengths");
+
+      // line search for relative error
+      // weighted median (weight = 1/target)
+      int nRows = endIdx - startIdx + 1; // startIdx and endIdx are inclusive
+      if (nRows == 1) return targetArr[idx[startIdx]] - predArr[idx[startIdx]]; // res 
+      else if (nRows == 2) {
+        // weighted average of two residuals
+        var w0 = Math.Abs(1.0 / targetArr[idx[startIdx]]);
+        var w1 = Math.Abs(1.0 / targetArr[idx[endIdx]]);
+        if (w0 > w1) {
+          return targetArr[idx[startIdx]] - predArr[idx[startIdx]];
+        } else if (w0 < w1) {
+          return targetArr[idx[endIdx]] - predArr[idx[endIdx]];
+        } else {
+          // same weight -> return average of both residuals
+          return ((targetArr[idx[startIdx]] - predArr[idx[startIdx]]) + (targetArr[idx[endIdx]] - predArr[idx[endIdx]])) / 2;
+        }
+      } else {
+        // create an array of key-value pairs to be sorted (instead of using Array.Sort(res, weights))
+        var res_w = new KeyValuePair<double, double>[nRows];
+        var totalWeight = 0.0;
+        for (int i = startIdx; i <= endIdx; i++) {
+          int row = idx[i];
+          var res = targetArr[row] - predArr[row];
+          var w = Math.Abs(1.0 / targetArr[row]);
+          res_w[i - startIdx] = new KeyValuePair<double, double>(res, w);
+          totalWeight += w;
+        }
+        // TODO: improve efficiency (find median without sort)
+        res_w.StableSort((a, b) => Math.Sign(a.Key - b.Key));
+
+        int k = 0;
+        double sum = totalWeight - res_w[k].Value; // total - first weight
+        while (sum > totalWeight / 2) {
+          k++;
+          sum -= res_w[k].Value;
+        }
+        return res_w[k].Key;
+      }
+    }
+
+    #region item implementation
+    [StorableConstructor]
+    private RelativeErrorLoss(StorableConstructorFlag _) : base(_) { }
+
+    private RelativeErrorLoss(RelativeErrorLoss original, Cloner cloner) : base(original, cloner) { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RelativeErrorLoss(this, cloner);
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/SquaredErrorLoss.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/SquaredErrorLoss.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/SquaredErrorLoss.cs	(revision 17041)
@@ -0,0 +1,89 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("5D02E552-B96E-4267-858B-22339D8CB6B2")]
+  [Item("Squared error loss", "")]
+  public sealed class SquaredErrorLoss : Item, ILossFunction {
+    public SquaredErrorLoss() { }
+
+    public double GetLoss(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+
+      double s = 0;
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        double res = targetEnum.Current - predEnum.Current;
+        s += res * res; // (res)^2
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have different lengths");
+
+      return s;
+    }
+
+    public IEnumerable<double> GetLossGradient(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        yield return 2.0 * (targetEnum.Current - predEnum.Current); // dL(y, f(x)) / df(x)  = 2 * res 
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have different lengths");
+    }
+
+    // targetArr and predArr are not changed by LineSearch
+    public double LineSearch(double[] targetArr, double[] predArr, int[] idx, int startIdx, int endIdx) {
+      if (targetArr.Length != predArr.Length)
+        throw new ArgumentException("target and pred have different lengths");
+
+      // line search for squared error loss 
+      // for a given partition of rows the optimal constant that should be added to the current prediction values is the average of the residuals
+      double s = 0.0;
+      int n = 0;
+      for (int i = startIdx; i <= endIdx; i++) {
+        int row = idx[i];
+        s += (targetArr[row] - predArr[row]);
+        n++;
+      }
+      return s / n;
+    }
+
+    #region item implementation
+    [StorableConstructor]
+    private SquaredErrorLoss(StorableConstructorFlag _) : base(_) { }
+
+    private SquaredErrorLoss(SquaredErrorLoss original, Cloner cloner) : base(original, cloner) { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new SquaredErrorLoss(this, cloner);
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/ModelCreation.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/ModelCreation.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/ModelCreation.cs	(revision 17041)
@@ -0,0 +1,39 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.GradientBoostedTrees {
+
+  /// <summary>
+  /// Defines what part of the Model should be stored.
+  /// QualityOnly - no solution will be created.
+  /// SurrogateModel - only the parameters will be stored, the model is calculated during deserialization
+  /// Model - the complete model will be stored (consider the amount of memory needed)
+  /// </summary>
+  [StorableType("EE55C357-C4B3-4662-B40B-D1D06A851809")]
+  public enum ModelCreation {
+    QualityOnly,
+    SurrogateModel,
+    Model
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeBuilder.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeBuilder.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeBuilder.cs	(revision 17041)
@@ -0,0 +1,473 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  // This class implements a greedy decision tree learner which selects splits with the maximum reduction in sum of squared errors.
+  // The tree builder also tracks variable relevance metrics based on the splits and improvement after the split.
+  // The implementation is tuned for gradient boosting where multiple trees have to be calculated for the same training data 
+  // each time with a different target vector. Vectors of idx to allow iteration of intput variables in sorted order are
+  // pre-calculated so that optimal thresholds for splits can be calculated in O(n) for each input variable.
+  // After each split the row idx are partitioned in a left an right part.
+  internal class RegressionTreeBuilder {
+    private readonly IRandom random;
+    private readonly IRegressionProblemData problemData;
+
+    private readonly int nCols;
+    private readonly double[][] x; // all training data (original order from problemData), x is constant
+    private double[] originalY; // the original target labels (from problemData), originalY is constant
+    private double[] curPred; // current predictions for originalY (in case we are using gradient boosting, otherwise = zeros), only necessary for line search
+
+    private double[] y; // training labels (original order from problemData), y can be changed
+
+    private Dictionary<string, double> sumImprovements; // for variable relevance calculation
+
+    private readonly string[] allowedVariables; // all variables in shuffled order
+    private Dictionary<string, int> varName2Index; // maps the variable names to column indexes 
+    private int effectiveVars; // number of variables that are used from allowedVariables
+
+    private int effectiveRows; // number of rows that are used from 
+    private readonly int[][] sortedIdxAll;
+    private readonly int[][] sortedIdx; // random selection from sortedIdxAll (for r < 1.0)
+
+    // helper arrays which are allocated to maximal necessary size only once in the ctor
+    private readonly int[] internalIdx, which, leftTmp, rightTmp;
+    private readonly double[] outx;
+    private readonly int[] outSortedIdx;
+
+    private RegressionTreeModel.TreeNode[] tree; // tree is represented as a flat array of nodes
+    private int curTreeNodeIdx; // the index where the next tree node is stored
+
+    // This class represents information about potential splits.
+    // For each node generated the best splitting variable and threshold as well as
+    // the improvement from the split are stored in a priority queue
+    private class PartitionSplits {
+      public int ParentNodeIdx { get; set; } // the idx of the leaf node representing this partition 
+      public int StartIdx { get; set; } // the start idx of the partition
+      public int EndIndex { get; set; } // the end idx of the partition
+      public string SplittingVariable { get; set; } // the best splitting variable
+      public double SplittingThreshold { get; set; } // the best threshold
+      public double SplittingImprovement { get; set; } // the improvement of the split (for priority queue)
+    }
+
+    // this list hold partitions with the information about the best split (organized as a sorted queue)
+    private readonly IList<PartitionSplits> queue;
+
+    // prepare and allocate buffer variables in ctor
+    public RegressionTreeBuilder(IRegressionProblemData problemData, IRandom random) {
+      this.problemData = problemData;
+      this.random = random;
+
+      var rows = problemData.TrainingIndices.Count();
+
+      this.nCols = problemData.AllowedInputVariables.Count();
+
+      allowedVariables = problemData.AllowedInputVariables.ToArray();
+      varName2Index = new Dictionary<string, int>(allowedVariables.Length);
+      for (int i = 0; i < allowedVariables.Length; i++) varName2Index.Add(allowedVariables[i], i);
+
+      sortedIdxAll = new int[nCols][];
+      sortedIdx = new int[nCols][];
+      sumImprovements = new Dictionary<string, double>();
+      internalIdx = new int[rows];
+      which = new int[rows];
+      leftTmp = new int[rows];
+      rightTmp = new int[rows];
+      outx = new double[rows];
+      outSortedIdx = new int[rows];
+      queue = new List<PartitionSplits>(100);
+
+      x = new double[nCols][];
+      originalY = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices).ToArray();
+      y = new double[originalY.Length];
+      Array.Copy(originalY, y, y.Length); // copy values (originalY is fixed, y is changed in gradient boosting)
+      curPred = Enumerable.Repeat(0.0, y.Length).ToArray(); // zeros
+
+      int col = 0;
+      foreach (var inputVariable in problemData.AllowedInputVariables) {
+        x[col] = problemData.Dataset.GetDoubleValues(inputVariable, problemData.TrainingIndices).ToArray();
+        sortedIdxAll[col] = Enumerable.Range(0, rows).OrderBy(r => x[col][r]).ToArray();
+        sortedIdx[col] = new int[rows];
+        col++;
+      }
+    }
+
+    // specific interface that allows to specify the target labels and the training rows which is necessary when for gradient boosted trees
+    public IRegressionModel CreateRegressionTreeForGradientBoosting(double[] y, double[] curPred, int maxSize, int[] idx, ILossFunction lossFunction, double r = 0.5, double m = 0.5) {
+      Debug.Assert(maxSize > 0);
+      Debug.Assert(r > 0);
+      Debug.Assert(r <= 1.0);
+      Debug.Assert(y.Count() == this.y.Length);
+      Debug.Assert(m > 0);
+      Debug.Assert(m <= 1.0);
+
+      // y and curPred are changed in gradient boosting
+      this.y = y;
+      this.curPred = curPred;
+
+      // shuffle row idx
+      HeuristicLab.Random.ListExtensions.ShuffleInPlace(idx, random);
+
+      int nRows = idx.Count();
+
+      // shuffle variable names
+      HeuristicLab.Random.ListExtensions.ShuffleInPlace(allowedVariables, random);
+
+      // only select a part of the rows and columns randomly
+      effectiveRows = (int)Math.Ceiling(nRows * r);
+      effectiveVars = (int)Math.Ceiling(nCols * m);
+
+      // the which array is used for partitioing row idxs  
+      Array.Clear(which, 0, which.Length);
+
+      // mark selected rows
+      for (int row = 0; row < effectiveRows; row++) {
+        which[idx[row]] = 1; // we use the which vector as a temporary variable here
+        internalIdx[row] = idx[row];
+      }
+
+      for (int col = 0; col < nCols; col++) {
+        int i = 0;
+        for (int row = 0; row < nRows; row++) {
+          if (which[sortedIdxAll[col][row]] > 0) {
+            Debug.Assert(i < effectiveRows);
+            sortedIdx[col][i] = sortedIdxAll[col][row];
+            i++;
+          }
+        }
+      }
+
+      this.tree = new RegressionTreeModel.TreeNode[maxSize];
+      this.queue.Clear();
+      this.curTreeNodeIdx = 0;
+
+      // start out with only one leaf node (constant prediction)
+      // and calculate the best split for this root node and enqueue it into a queue sorted by improvement throught the split
+      // start and end idx are inclusive
+      CreateLeafNode(0, effectiveRows - 1, lossFunction);
+
+      // process the priority queue to complete the tree
+      CreateRegressionTreeFromQueue(maxSize, lossFunction);
+
+      return new RegressionTreeModel(tree.ToArray(), problemData.TargetVariable);
+    }
+
+
+    // processes potential splits from the queue as long as splits are remaining and the maximum size of the tree is not reached
+    private void CreateRegressionTreeFromQueue(int maxNodes, ILossFunction lossFunction) {
+      while (queue.Any() && curTreeNodeIdx + 1 < maxNodes) { // two nodes are created in each loop
+        var f = queue[queue.Count - 1]; // last element has the largest improvement
+        queue.RemoveAt(queue.Count - 1);
+
+        var startIdx = f.StartIdx;
+        var endIdx = f.EndIndex;
+
+        Debug.Assert(endIdx - startIdx >= 0);
+        Debug.Assert(startIdx >= 0);
+        Debug.Assert(endIdx < internalIdx.Length);
+
+        // split partition into left and right
+        int splitIdx;
+        SplitPartition(f.StartIdx, f.EndIndex, f.SplittingVariable, f.SplittingThreshold, out splitIdx);
+        Debug.Assert(splitIdx + 1 <= endIdx);
+        Debug.Assert(startIdx <= splitIdx);
+
+        // create two leaf nodes (and enqueue best splits for both)
+        var leftTreeIdx = CreateLeafNode(startIdx, splitIdx, lossFunction);
+        var rightTreeIdx = CreateLeafNode(splitIdx + 1, endIdx, lossFunction);
+
+        // overwrite existing leaf node with an internal node
+        tree[f.ParentNodeIdx] = new RegressionTreeModel.TreeNode(f.SplittingVariable, f.SplittingThreshold, leftTreeIdx, rightTreeIdx, weightLeft: (splitIdx - startIdx + 1) / (double)(endIdx - startIdx + 1));
+      }
+    }
+
+
+    // returns the index of the newly created tree node
+    private int CreateLeafNode(int startIdx, int endIdx, ILossFunction lossFunction) {
+      // write a leaf node
+      var val = lossFunction.LineSearch(originalY, curPred, internalIdx, startIdx, endIdx);
+      tree[curTreeNodeIdx] = new RegressionTreeModel.TreeNode(RegressionTreeModel.TreeNode.NO_VARIABLE, val);
+
+      EnqueuePartitionSplit(curTreeNodeIdx, startIdx, endIdx);
+      curTreeNodeIdx++;
+      return curTreeNodeIdx - 1;
+    }
+
+
+    // calculates the optimal split for the partition [startIdx .. endIdx] (inclusive) 
+    // which is represented by the leaf node with the specified nodeIdx
+    private void EnqueuePartitionSplit(int nodeIdx, int startIdx, int endIdx) {
+      double threshold, improvement;
+      string bestVariableName;
+      // only enqueue a new split if there are at least 2 rows left and a split is possible
+      if (startIdx < endIdx &&
+        FindBestVariableAndThreshold(startIdx, endIdx, out threshold, out bestVariableName, out improvement)) {
+        var split = new PartitionSplits() {
+          ParentNodeIdx = nodeIdx,
+          StartIdx = startIdx,
+          EndIndex = endIdx,
+          SplittingThreshold = threshold,
+          SplittingVariable = bestVariableName
+        };
+        InsertSortedQueue(split);
+      }
+    }
+
+
+    // routine for splitting a partition of rows stored in internalIdx between startIdx and endIdx into
+    // a left partition and a right partition using the given splittingVariable and threshold
+    // the splitIdx is the last index of the left partition 
+    // splitIdx + 1 is the first index of the right partition
+    // startIdx and endIdx are inclusive
+    private void SplitPartition(int startIdx, int endIdx, string splittingVar, double threshold, out int splitIdx) {
+      int bestVarIdx = varName2Index[splittingVar];
+      // split - two pass
+
+      // store which index goes into which partition 
+      for (int k = startIdx; k <= endIdx; k++) {
+        if (x[bestVarIdx][internalIdx[k]] <= threshold)
+          which[internalIdx[k]] = -1; // left partition
+        else
+          which[internalIdx[k]] = 1; // right partition
+      }
+
+      // partition sortedIdx for each variable
+      int i;
+      int j;
+      for (int col = 0; col < nCols; col++) {
+        i = 0;
+        j = 0;
+        int k;
+        for (k = startIdx; k <= endIdx; k++) {
+          Debug.Assert(Math.Abs(which[sortedIdx[col][k]]) == 1);
+
+          if (which[sortedIdx[col][k]] < 0) {
+            leftTmp[i++] = sortedIdx[col][k];
+          } else {
+            rightTmp[j++] = sortedIdx[col][k];
+          }
+        }
+        Debug.Assert(i > 0); // at least on element in the left partition
+        Debug.Assert(j > 0); // at least one element in the right partition
+        Debug.Assert(i + j == endIdx - startIdx + 1);
+        k = startIdx;
+        for (int l = 0; l < i; l++) sortedIdx[col][k++] = leftTmp[l];
+        for (int l = 0; l < j; l++) sortedIdx[col][k++] = rightTmp[l];
+      }
+
+      // partition row indices
+      i = startIdx;
+      j = endIdx;
+      while (i <= j) {
+        Debug.Assert(Math.Abs(which[internalIdx[i]]) == 1);
+        Debug.Assert(Math.Abs(which[internalIdx[j]]) == 1);
+        if (which[internalIdx[i]] < 0) i++;
+        else if (which[internalIdx[j]] > 0) j--;
+        else {
+          Debug.Assert(which[internalIdx[i]] > 0);
+          Debug.Assert(which[internalIdx[j]] < 0);
+          // swap
+          int tmp = internalIdx[i];
+          internalIdx[i] = internalIdx[j];
+          internalIdx[j] = tmp;
+          i++;
+          j--;
+        }
+      }
+      Debug.Assert(j + 1 == i);
+      Debug.Assert(i <= endIdx);
+      Debug.Assert(startIdx <= j);
+
+      splitIdx = j;
+    }
+
+    private bool FindBestVariableAndThreshold(int startIdx, int endIdx, out double threshold, out string bestVar, out double improvement) {
+      Debug.Assert(startIdx < endIdx + 1); // at least 2 elements
+
+      int rows = endIdx - startIdx + 1;
+      Debug.Assert(rows >= 2);
+
+      double sumY = 0.0;
+      for (int i = startIdx; i <= endIdx; i++) {
+        sumY += y[internalIdx[i]];
+      }
+
+      // see description of calculation in FindBestThreshold
+      double bestImprovement = 1.0 / rows * sumY * sumY; // any improvement must be larger than this baseline
+      double bestThreshold = double.PositiveInfinity;
+      bestVar = RegressionTreeModel.TreeNode.NO_VARIABLE;
+
+      for (int col = 0; col < effectiveVars; col++) {
+        // sort values for variable to prepare for threshold selection 
+        var curVariable = allowedVariables[col];
+        var curVariableIdx = varName2Index[curVariable];
+        for (int i = startIdx; i <= endIdx; i++) {
+          var sortedI = sortedIdx[curVariableIdx][i];
+          outSortedIdx[i - startIdx] = sortedI;
+          outx[i - startIdx] = x[curVariableIdx][sortedI];
+        }
+
+        double curImprovement;
+        double curThreshold;
+        FindBestThreshold(outx, outSortedIdx, rows, y, sumY, out curThreshold, out curImprovement);
+
+        if (curImprovement > bestImprovement) {
+          bestImprovement = curImprovement;
+          bestThreshold = curThreshold;
+          bestVar = allowedVariables[col];
+        }
+      }
+      if (bestVar == RegressionTreeModel.TreeNode.NO_VARIABLE) {
+        // not successfull
+        threshold = double.PositiveInfinity;
+        improvement = double.NegativeInfinity;
+        return false;
+      } else {
+        UpdateVariableRelevance(bestVar, sumY, bestImprovement, rows);
+        improvement = bestImprovement;
+        threshold = bestThreshold;
+        return true;
+      }
+    }
+
+    // x [0..N-1] contains rows sorted values in the range from [0..rows-1]
+    // sortedIdx [0..N-1] contains the idx of the values in x in the original dataset in the range from [0..rows-1]
+    // rows specifies the number of valid entries in x and sortedIdx
+    // y [0..N-1] contains the target values in original sorting order
+    // sumY is y.Sum()
+    // 
+    // the routine returns the best threshold (x[i] + x[i+1]) / 2 for i = [0 .. rows-2] by calculating the reduction in squared error
+    // additionally the reduction in squared error is returned in bestImprovement
+    // if all elements of x are equal the routing fails to produce a threshold
+    private static void FindBestThreshold(double[] x, int[] sortedIdx, int rows, double[] y, double sumY, out double bestThreshold, out double bestImprovement) {
+      Debug.Assert(rows >= 2);
+
+      double sl = 0.0;
+      double sr = sumY;
+      double nl = 0.0;
+      double nr = rows;
+
+      bestImprovement = 1.0 / rows * sumY * sumY; // this is the baseline for the improvement
+      bestThreshold = double.NegativeInfinity;
+      // for all thresholds
+      // if we have n rows there are n-1 possible splits
+      for (int i = 0; i < rows - 1; i++) {
+        sl += y[sortedIdx[i]];
+        sr -= y[sortedIdx[i]];
+
+        nl++;
+        nr--;
+        Debug.Assert(nl > 0);
+        Debug.Assert(nr > 0);
+
+        if (x[i] < x[i + 1]) { // don't try to split when two elements are equal
+
+          // goal is to find the split with leading to minimal total variance of left and right parts
+          // without partitioning the variance is var(y) = E(y²) - E(y)²  
+          //    = 1/n * sum(y²) - (1/n * sum(y))²
+          //      -------------   ---------------
+          //         constant       baseline for improvement
+          // 
+          // if we split into right and left part the overall variance is the weigthed combination nl/n * var(y_l) + nr/n * var(y_r)  
+          //    = nl/n * (1/nl * sum(y_l²) - (1/nl * sum(y_l))²) + nr/n * (1/nr * sum(y_r²) - (1/nr * sum(y_r))²)
+          //    = 1/n * sum(y_l²) - 1/nl * 1/n * sum(y_l)² + 1/n * sum(y_r²) - 1/nr * 1/n * sum(y_r)²
+          //    = 1/n * (sum(y_l²) + sum(y_r²)) - 1/n * (sum(y_l)² / nl + sum(y_r)² / nr)
+          //    = 1/n * sum(y²) - 1/n * (sum(y_l)² / nl + sum(y_r)² / nr)
+          //      -------------
+          //       not changed by split (and the same for total variance without partitioning)
+          //
+          //   therefore we need to find the maximum value (sum(y_l)² / nl + sum(y_r)² / nr) (ignoring the factor 1/n)
+          //   and this value must be larger than 1/n * sum(y)² to be an improvement over no split 
+
+          double curQuality = sl * sl / nl + sr * sr / nr;
+
+          if (curQuality > bestImprovement) {
+            bestThreshold = (x[i] + x[i + 1]) / 2.0;
+            bestImprovement = curQuality;
+          }
+        }
+      }
+
+      // if all elements where the same then no split can be found
+    }
+
+
+    private void UpdateVariableRelevance(string bestVar, double sumY, double bestImprovement, int rows) {
+      if (string.IsNullOrEmpty(bestVar)) return;
+      // update variable relevance
+      double baseLine = 1.0 / rows * sumY * sumY; // if best improvement is equal to baseline then the split had no effect
+
+      double delta = (bestImprovement - baseLine);
+      double v;
+      if (!sumImprovements.TryGetValue(bestVar, out v)) {
+        sumImprovements[bestVar] = delta;
+      }
+      sumImprovements[bestVar] = v + delta;
+    }
+
+    public IEnumerable<KeyValuePair<string, double>> GetVariableRelevance() {
+      // values are scaled: the most important variable has relevance = 100
+      double scaling = 100 / sumImprovements.Max(t => t.Value);
+      return
+        sumImprovements
+        .Select(t => new KeyValuePair<string, double>(t.Key, t.Value * scaling))
+        .OrderByDescending(t => t.Value);
+    }
+
+
+    // insert a new parition split (find insertion point and start at first element of the queue)
+    // elements are removed from the queue at the last position
+    // O(n), splits could be organized as a heap to improve runtime (see alglib tsort)
+    private void InsertSortedQueue(PartitionSplits split) {
+      // find insertion position
+      int i = 0;
+      while (i < queue.Count && queue[i].SplittingImprovement < split.SplittingImprovement) { i++; }
+
+      queue.Insert(i, split);
+    }
+  }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeModel.cs	(revision 17041)
@@ -0,0 +1,273 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Globalization;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("C383410E-8707-486F-98F6-1DFB708B09B5")]
+  [Item("RegressionTreeModel", "Represents a decision tree for regression.")]
+  public sealed class RegressionTreeModel : RegressionModel {
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return tree.Select(t => t.VarName).Where(v => v != TreeNode.NO_VARIABLE); }
+    }
+
+    // trees are represented as a flat array    
+    internal struct TreeNode {
+      public readonly static string NO_VARIABLE = null;
+
+      public TreeNode(string varName, double val, int leftIdx = -1, int rightIdx = -1, double weightLeft = -1.0)
+        : this() {
+        VarName = varName;
+        Val = val;
+        LeftIdx = leftIdx;
+        RightIdx = rightIdx;
+        WeightLeft = weightLeft;
+      }
+
+      public string VarName { get; internal set; } // name of the variable for splitting or NO_VARIABLE if terminal node
+      public double Val { get; internal set; } // threshold
+      public int LeftIdx { get; internal set; }
+      public int RightIdx { get; internal set; }
+      public double WeightLeft { get; internal set; } // for partial dependence plots (value in range [0..1] describes the fraction of training samples for the left sub-tree
+
+
+      // necessary because the default implementation of GetHashCode for structs in .NET would only return the hashcode of val here
+      public override int GetHashCode() {
+        return LeftIdx ^ RightIdx ^ Val.GetHashCode();
+      }
+      // necessary because of GetHashCode override
+      public override bool Equals(object obj) {
+        if (obj is TreeNode) {
+          var other = (TreeNode)obj;
+          return Val.Equals(other.Val) &&
+            LeftIdx.Equals(other.LeftIdx) &&
+            RightIdx.Equals(other.RightIdx) &&
+            WeightLeft.Equals(other.WeightLeft) &&
+            EqualStrings(VarName, other.VarName);
+        } else {
+          return false;
+        }
+      }
+
+      private bool EqualStrings(string a, string b) {
+        return (a == null && b == null) ||
+               (a != null && b != null && a.Equals(b));
+      }
+    }
+
+    // not storable!
+    private TreeNode[] tree;
+
+    #region old storable format
+    // remove with HL 3.4
+    [Storable(OldName = "SerializedTree")]
+    // to prevent storing the references to data caches in nodes
+    // seemingly, it is bad (performance-wise) to persist tuples (tuples are used as keys in a dictionary)
+    private Tuple<string, double, int, int>[] SerializedTree {
+      // get { return tree.Select(t => Tuple.Create(t.VarName, t.Val, t.LeftIdx, t.RightIdx)).ToArray(); }
+      set { this.tree = value.Select(t => new TreeNode(t.Item1, t.Item2, t.Item3, t.Item4, -1.0)).ToArray(); } // use a weight of -1.0 to indicate that partial dependence cannot be calculated for old models 
+    }
+    #endregion
+    #region new storable format
+    [Storable]
+    private string[] SerializedTreeVarNames {
+      get { return tree.Select(t => t.VarName).ToArray(); }
+      set {
+        if (tree == null) tree = new TreeNode[value.Length];
+        for (int i = 0; i < value.Length; i++) {
+          tree[i].VarName = value[i];
+        }
+      }
+    }
+    [Storable]
+    private double[] SerializedTreeValues {
+      get { return tree.Select(t => t.Val).ToArray(); }
+      set {
+        if (tree == null) tree = new TreeNode[value.Length];
+        for (int i = 0; i < value.Length; i++) {
+          tree[i].Val = value[i];
+        }
+      }
+    }
+    [Storable]
+    private int[] SerializedTreeLeftIdx {
+      get { return tree.Select(t => t.LeftIdx).ToArray(); }
+      set {
+        if (tree == null) tree = new TreeNode[value.Length];
+        for (int i = 0; i < value.Length; i++) {
+          tree[i].LeftIdx = value[i];
+        }
+      }
+    }
+    [Storable]
+    private int[] SerializedTreeRightIdx {
+      get { return tree.Select(t => t.RightIdx).ToArray(); }
+      set {
+        if (tree == null) tree = new TreeNode[value.Length];
+        for (int i = 0; i < value.Length; i++) {
+          tree[i].RightIdx = value[i];
+        }
+      }
+    }
+    [Storable]
+    private double[] SerializedTreeWeightLeft {
+      get { return tree.Select(t => t.WeightLeft).ToArray(); }
+      set {
+        if (tree == null) tree = new TreeNode[value.Length];
+        for (int i = 0; i < value.Length; i++) {
+          tree[i].WeightLeft = value[i];
+        }
+      }
+    }
+    #endregion
+
+    [StorableConstructor]
+    private RegressionTreeModel(StorableConstructorFlag _) : base(_) { }
+    // cloning ctor
+    private RegressionTreeModel(RegressionTreeModel original, Cloner cloner)
+      : base(original, cloner) {
+      if (original.tree != null) {
+        this.tree = new TreeNode[original.tree.Length];
+        Array.Copy(original.tree, this.tree, this.tree.Length);
+      }
+    }
+
+    internal RegressionTreeModel(TreeNode[] tree, string targetVariable)
+      : base(targetVariable, "RegressionTreeModel", "Represents a decision tree for regression.") {
+      this.tree = tree;
+    }
+
+    private static double GetPredictionForRow(TreeNode[] t, ReadOnlyCollection<double>[] columnCache, int nodeIdx, int row) {
+      while (nodeIdx != -1) {
+        var node = t[nodeIdx];
+        if (node.VarName == TreeNode.NO_VARIABLE)
+          return node.Val;
+        if (columnCache[nodeIdx] == null || double.IsNaN(columnCache[nodeIdx][row])) {
+          if (node.WeightLeft.IsAlmost(-1.0)) throw new InvalidOperationException("Cannot calculate partial dependence for trees loaded from older versions of HeuristicLab.");
+          // weighted average for partial dependence plot (recursive here because we need to calculate both sub-trees)
+          return node.WeightLeft * GetPredictionForRow(t, columnCache, node.LeftIdx, row) +
+                 (1.0 - node.WeightLeft) * GetPredictionForRow(t, columnCache, node.RightIdx, row);
+        } else if (columnCache[nodeIdx][row] <= node.Val)
+          nodeIdx = node.LeftIdx;
+        else
+          nodeIdx = node.RightIdx;
+      }
+      throw new InvalidOperationException("Invalid tree in RegressionTreeModel");
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RegressionTreeModel(this, cloner);
+    }
+
+    public override IEnumerable<double> GetEstimatedValues(IDataset ds, IEnumerable<int> rows) {
+      // lookup columns for variableNames in one pass over the tree to speed up evaluation later on
+      ReadOnlyCollection<double>[] columnCache = new ReadOnlyCollection<double>[tree.Length];
+
+      for (int i = 0; i < tree.Length; i++) {
+        if (tree[i].VarName != TreeNode.NO_VARIABLE) {
+          // tree models also support calculating estimations if not all variables used for training are available in the dataset
+          if (ds.ColumnNames.Contains(tree[i].VarName))
+            columnCache[i] = ds.GetReadOnlyDoubleValues(tree[i].VarName);
+        }
+      }
+      return rows.Select(r => GetPredictionForRow(tree, columnCache, 0, r));
+    }
+
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RegressionSolution(this, new RegressionProblemData(problemData));
+    }
+
+    // mainly for debugging
+    public override string ToString() {
+      return TreeToString(0, "");
+    }
+
+    /// <summary>
+    /// Transforms the tree model to a symbolic regression solution
+    /// </summary>
+    /// <param name="problemData"></param>
+    /// <returns>A new symbolic regression solution which matches the tree model</returns>
+    public ISymbolicRegressionSolution CreateSymbolicRegressionSolution(IRegressionProblemData problemData) {
+      return CreateSymbolicRegressionModel().CreateRegressionSolution(problemData);
+    }
+
+    /// <summary>
+    /// Transforms the tree model to a symbolic regression model
+    /// </summary>
+    /// <returns>A new symbolic regression model which matches the tree model</returns>
+    public SymbolicRegressionModel CreateSymbolicRegressionModel() {
+      var rootSy = new ProgramRootSymbol();
+      var startSy = new StartSymbol();
+      var varCondSy = new VariableCondition() { IgnoreSlope = true };
+      var constSy = new Constant();
+
+      var startNode = startSy.CreateTreeNode();
+      startNode.AddSubtree(CreateSymbolicRegressionTreeRecursive(tree, 0, varCondSy, constSy));
+      var rootNode = rootSy.CreateTreeNode();
+      rootNode.AddSubtree(startNode);
+      return new SymbolicRegressionModel(TargetVariable, new SymbolicExpressionTree(rootNode), new SymbolicDataAnalysisExpressionTreeLinearInterpreter());
+    }
+
+    private ISymbolicExpressionTreeNode CreateSymbolicRegressionTreeRecursive(TreeNode[] treeNodes, int nodeIdx, VariableCondition varCondSy, Constant constSy) {
+      var curNode = treeNodes[nodeIdx];
+      if (curNode.VarName == TreeNode.NO_VARIABLE) {
+        var node = (ConstantTreeNode)constSy.CreateTreeNode();
+        node.Value = curNode.Val;
+        return node;
+      } else {
+        var node = (VariableConditionTreeNode)varCondSy.CreateTreeNode();
+        node.VariableName = curNode.VarName;
+        node.Threshold = curNode.Val;
+
+        var left = CreateSymbolicRegressionTreeRecursive(treeNodes, curNode.LeftIdx, varCondSy, constSy);
+        var right = CreateSymbolicRegressionTreeRecursive(treeNodes, curNode.RightIdx, varCondSy, constSy);
+        node.AddSubtree(left);
+        node.AddSubtree(right);
+        return node;
+      }
+    }
+
+
+    private string TreeToString(int idx, string part) {
+      var n = tree[idx];
+      if (n.VarName == TreeNode.NO_VARIABLE) {
+        return string.Format(CultureInfo.InvariantCulture, "{0} -> {1:F}{2}", part, n.Val, Environment.NewLine);
+      } else {
+        return
+          TreeToString(n.LeftIdx, string.Format(CultureInfo.InvariantCulture, "{0}{1}{2} <= {3:F} ({4:N3})", part, string.IsNullOrEmpty(part) ? "" : " and ", n.VarName, n.Val, n.WeightLeft))
+        + TreeToString(n.RightIdx, string.Format(CultureInfo.InvariantCulture, "{0}{1}{2}  >  {3:F} ({4:N3}))", part, string.IsNullOrEmpty(part) ? "" : " and ", n.VarName, n.Val, 1.0 - n.WeightLeft));
+      }
+    }
+
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj	(revision 17041)
@@ -0,0 +1,500 @@
+﻿<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" ToolsVersion="4.0">
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProductVersion>9.0.30729</ProductVersion>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{2E782078-FA81-4B70-B56F-74CE38DAC6C8}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>HeuristicLab.Algorithms.DataAnalysis</RootNamespace>
+    <AssemblyName>HeuristicLab.Algorithms.DataAnalysis-3.4</AssemblyName>
+    <SignAssembly>true</SignAssembly>
+    <AssemblyOriginatorKeyFile>HeuristicLab.snk</AssemblyOriginatorKeyFile>
+    <FileUpgradeFlags>
+    </FileUpgradeFlags>
+    <OldToolsVersion>3.5</OldToolsVersion>
+    <UpgradeBackupLocation>
+    </UpgradeBackupLocation>
+    <StartupObject>
+    </StartupObject>
+    <TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
+    <TargetFrameworkProfile>
+    </TargetFrameworkProfile>
+    <PublishUrl>publish\</PublishUrl>
+    <Install>true</Install>
+    <InstallFrom>Disk</InstallFrom>
+    <UpdateEnabled>false</UpdateEnabled>
+    <UpdateMode>Foreground</UpdateMode>
+    <UpdateInterval>7</UpdateInterval>
+    <UpdateIntervalUnits>Days</UpdateIntervalUnits>
+    <UpdatePeriodically>false</UpdatePeriodically>
+    <UpdateRequired>false</UpdateRequired>
+    <MapFileExtensions>true</MapFileExtensions>
+    <ApplicationRevision>0</ApplicationRevision>
+    <ApplicationVersion>1.0.0.%2a</ApplicationVersion>
+    <IsWebBootstrapper>false</IsWebBootstrapper>
+    <UseApplicationTrust>false</UseApplicationTrust>
+    <BootstrapperEnabled>true</BootstrapperEnabled>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>$(SolutionDir)\bin\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <Prefer32Bit>false</Prefer32Bit>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>$(SolutionDir)\bin\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <DocumentationFile>
+    </DocumentationFile>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <Prefer32Bit>false</Prefer32Bit>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x86' ">
+    <DebugSymbols>true</DebugSymbols>
+    <OutputPath>$(SolutionDir)\bin\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <DebugType>full</DebugType>
+    <PlatformTarget>x86</PlatformTarget>
+    <ErrorReport>prompt</ErrorReport>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <Prefer32Bit>false</Prefer32Bit>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x86' ">
+    <OutputPath>$(SolutionDir)\bin\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <DocumentationFile>
+    </DocumentationFile>
+    <Optimize>true</Optimize>
+    <DebugType>pdbonly</DebugType>
+    <PlatformTarget>x86</PlatformTarget>
+    <ErrorReport>prompt</ErrorReport>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <Prefer32Bit>false</Prefer32Bit>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
+    <DebugSymbols>true</DebugSymbols>
+    <OutputPath>$(SolutionDir)\bin\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <DebugType>full</DebugType>
+    <PlatformTarget>x64</PlatformTarget>
+    <ErrorReport>prompt</ErrorReport>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <Prefer32Bit>false</Prefer32Bit>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
+    <OutputPath>$(SolutionDir)\bin\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <DocumentationFile>
+    </DocumentationFile>
+    <Optimize>true</Optimize>
+    <DebugType>pdbonly</DebugType>
+    <PlatformTarget>x64</PlatformTarget>
+    <ErrorReport>prompt</ErrorReport>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <Prefer32Bit>false</Prefer32Bit>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="ALGLIB-3.7.0, Version=3.7.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\bin\ALGLIB-3.7.0.dll</HintPath>
+      <Private>False</Private>
+    </Reference>
+    <Reference Include="LibSVM-3.12, Version=3.12.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\bin\LibSVM-3.12.dll</HintPath>
+      <Private>False</Private>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core">
+      <RequiredTargetFramework>3.5</RequiredTargetFramework>
+    </Reference>
+    <Reference Include="System.Drawing" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="BaselineClassifiers\OneFactorClassificationModel.cs" />
+    <Compile Include="BaselineClassifiers\OneFactorClassificationSolution.cs" />
+    <Compile Include="BaselineClassifiers\OneR.cs" />
+    <Compile Include="BaselineClassifiers\OneRClassificationModel.cs" />
+    <Compile Include="BaselineClassifiers\OneRClassificationSolution.cs" />
+    <Compile Include="BaselineClassifiers\ZeroR.cs" />
+    <Compile Include="CrossValidation.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="DoubleArrayExtensions.cs" />
+    <Compile Include="FixedDataAnalysisAlgorithm.cs" />
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceSpectralMixture.cs" />
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovariancePiecewisePolynomial.cs" />
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceNeuralNetwork.cs" />
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovariancePolynomial.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessBase.cs" />
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceConst.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceLinear.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceLinearArd.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceMask.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceMaternIso.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceNoise.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovariancePeriodic.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceProduct.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceRationalQuadraticArd.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceRationalQuadraticIso.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceScale.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceSquaredExponentialArd.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceSquaredExponentialIso.cs" />
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceSum.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\GaussianProcessClassificationSolutionCreator.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessClassificationModelCreator.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessClassification.cs" />
+    <Compile Include="GaussianProcess\MeanFunctions\MeanModel.cs" />
+    <Compile Include="GaussianProcess\MeanFunctions\MeanConst.cs" />
+    <Compile Include="GaussianProcess\MeanFunctions\MeanLinear.cs" />
+    <Compile Include="GaussianProcess\MeanFunctions\MeanProduct.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\MeanFunctions\MeanSum.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\MeanFunctions\MeanZero.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessHyperparameterInitializer.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessRegressionSolutionCreator.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessRegressionModelCreator.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessModelCreator.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\GaussianProcessCovarianceOptimizationProblem.cs" />
+    <Compile Include="GaussianProcess\Util.cs" />
+    <Compile Include="GaussianProcess\IMeanFunction.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessModel.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessRegression.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessRegressionSolution.cs" />
+    <Compile Include="GaussianProcess\ICovarianceFunction.cs" />
+    <Compile Include="GBM\GradientBoostingRegressionAlgorithm.cs" />
+    <Compile Include="GradientBoostedTrees\IGradientBoostedTreesModel.cs" />
+    <Compile Include="GradientBoostedTrees\GradientBoostedTreesModelSurrogate.cs" />
+    <Compile Include="GradientBoostedTrees\GradientBoostedTreesAlgorithm.cs" />
+    <Compile Include="GradientBoostedTrees\GradientBoostedTreesAlgorithmStatic.cs" />
+    <Compile Include="GradientBoostedTrees\GradientBoostedTreesModel.cs" />
+    <Compile Include="GradientBoostedTrees\LossFunctions\AbsoluteErrorLoss.cs" />
+    <Compile Include="GradientBoostedTrees\LossFunctions\QuantileRegressionLoss.cs" />
+    <Compile Include="GradientBoostedTrees\LossFunctions\ILossFunction.cs" />
+    <Compile Include="GradientBoostedTrees\LossFunctions\LogisticRegressionLoss.cs" />
+    <Compile Include="GradientBoostedTrees\LossFunctions\RelativeErrorLoss.cs" />
+    <Compile Include="GradientBoostedTrees\LossFunctions\SquaredErrorLoss.cs" />
+    <Compile Include="GradientBoostedTrees\GradientBoostedTreesSolution.cs" />
+    <Compile Include="GradientBoostedTrees\ModelCreation.cs" />
+    <Compile Include="GradientBoostedTrees\RegressionTreeBuilder.cs" />
+    <Compile Include="GradientBoostedTrees\RegressionTreeModel.cs" />
+    <Compile Include="Interfaces\IGaussianProcessClassificationModelCreator.cs" />
+    <Compile Include="Interfaces\IGaussianProcessRegressionModelCreator.cs" />
+    <Compile Include="Interfaces\IGaussianProcessModelCreator.cs" />
+    <Compile Include="Interfaces\IGaussianProcessModel.cs" />
+    <Compile Include="Interfaces\IGaussianProcessSolution.cs" />
+    <Compile Include="Interfaces\INcaClassificationSolution.cs" />
+    <Compile Include="Interfaces\INcaModel.cs" />
+    <Compile Include="Interfaces\INearestNeighbourClassificationSolution.cs" />
+    <Compile Include="Interfaces\INearestNeighbourRegressionSolution.cs" />
+    <Compile Include="Interfaces\INearestNeighbourModel.cs" />
+    <Compile Include="Interfaces\INeuralNetworkEnsembleClassificationSolution.cs" />
+    <Compile Include="Interfaces\INeuralNetworkEnsembleRegressionSolution.cs" />
+    <Compile Include="Interfaces\INeuralNetworkEnsembleModel.cs" />
+    <Compile Include="Interfaces\INeuralNetworkClassificationSolution.cs" />
+    <Compile Include="Interfaces\INeuralNetworkRegressionSolution.cs" />
+    <Compile Include="Interfaces\INeuralNetworkModel.cs" />
+    <Compile Include="Interfaces\IRandomForestClassificationSolution.cs" />
+    <Compile Include="Interfaces\IRandomForestModel.cs" />
+    <Compile Include="Interfaces\IRandomForestRegressionSolution.cs" />
+    <Compile Include="Interfaces\ISupportVectorMachineModel.cs" />
+    <Compile Include="Interfaces\ISupportVectorMachineSolution.cs" />
+    <Compile Include="Interfaces\IDataAnalysisAlgorithm.cs" />
+    <Compile Include="KernelRidgeRegression\KernelFunctions\CicularKernel.cs" />
+    <Compile Include="KernelRidgeRegression\KernelFunctions\GaussianKernel.cs" />
+    <Compile Include="KernelRidgeRegression\KernelFunctions\IKernel.cs" />
+    <Compile Include="KernelRidgeRegression\KernelFunctions\InverseMultiquadraticKernel.cs" />
+    <Compile Include="KernelRidgeRegression\KernelFunctions\KernelBase.cs" />
+    <Compile Include="KernelRidgeRegression\KernelFunctions\MultiquadraticKernel.cs" />
+    <Compile Include="KernelRidgeRegression\KernelFunctions\PolysplineKernel.cs" />
+    <Compile Include="KernelRidgeRegression\KernelFunctions\ThinPlatePolysplineKernel.cs" />
+    <Compile Include="KernelRidgeRegression\KernelRidgeRegression.cs" />
+    <Compile Include="KernelRidgeRegression\KernelRidgeRegressionModel.cs" />
+    <Compile Include="kMeans\KMeansClustering.cs" />
+    <Compile Include="kMeans\KMeansClusteringModel.cs" />
+    <Compile Include="kMeans\KMeansClusteringSolution.cs" />
+    <Compile Include="kMeans\KMeansClusteringUtil.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="Linear\LinearDiscriminantAnalysis.cs" />
+    <Compile Include="Linear\LinearRegression.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="Linear\MultinomialLogitClassification.cs" />
+    <Compile Include="Linear\MultinomialLogitClassificationSolution.cs" />
+    <Compile Include="Linear\LinearRegressionModel.cs" />
+    <Compile Include="Linear\MultinomialLogitModel.cs" />
+    <Compile Include="Linear\Scaling.cs" />
+    <Compile Include="Nca\Initialization\INcaInitializer.cs" />
+    <Compile Include="Nca\Initialization\LdaInitializer.cs" />
+    <Compile Include="Nca\Initialization\NcaInitializer.cs" />
+    <Compile Include="Nca\Initialization\PcaInitializer.cs" />
+    <Compile Include="Nca\Initialization\RandomInitializer.cs" />
+    <Compile Include="Nca\Matrix.cs" />
+    <Compile Include="Nca\ModelCreation\INcaModelCreator.cs" />
+    <Compile Include="Nca\ModelCreation\NcaModelCreator.cs" />
+    <Compile Include="Nca\NcaAlgorithm.cs" />
+    <Compile Include="Nca\NcaClassificationSolution.cs" />
+    <Compile Include="Nca\NcaGradientCalculator.cs" />
+    <Compile Include="Nca\NcaModel.cs" />
+    <Compile Include="Nca\SolutionCreation\INcaSolutionCreator.cs" />
+    <Compile Include="Nca\SolutionCreation\NcaSolutionCreator.cs" />
+    <Compile Include="NearestNeighbour\NearestNeighbourClassification.cs" />
+    <Compile Include="NearestNeighbour\NearestNeighbourClassificationSolution.cs" />
+    <Compile Include="NearestNeighbour\NearestNeighbourModel.cs" />
+    <Compile Include="NearestNeighbour\NearestNeighbourRegression.cs" />
+    <Compile Include="NearestNeighbour\NearestNeighbourRegressionSolution.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkEnsembleClassification.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkEnsembleClassificationSolution.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkEnsembleModel.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkEnsembleRegressionSolution.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkEnsembleRegression.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkClassification.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkClassificationSolution.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkModel.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkRegression.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkRegressionSolution.cs" />
+    <Compile Include="NonlinearRegression\NonlinearRegression.cs" />
+    <Compile Include="Plugin.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="RandomForest\RandomForestClassificationSolution.cs" />
+    <Compile Include="RandomForest\RandomForestClassification.cs" />
+    <Compile Include="RandomForest\RandomForestModel.cs" />
+    <Compile Include="RandomForest\RandomForestRegression.cs" />
+    <Compile Include="RandomForest\RandomForestRegressionSolution.cs" />
+    <Compile Include="RandomForest\RandomForestUtil.cs" />
+    <Compile Include="SupportVectorMachine\SupportVectorClassification.cs" />
+    <Compile Include="SupportVectorMachine\SupportVectorClassificationSolution.cs" />
+    <Compile Include="SupportVectorMachine\SupportVectorMachineModel.cs" />
+    <Compile Include="SupportVectorMachine\SupportVectorMachineUtil.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="SupportVectorMachine\SupportVectorRegression.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="SupportVectorMachine\SupportVectorRegressionSolution.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="TimeSeries\AutoregressiveModeling.cs" />
+    <Compile Include="TSNE\Distances\CosineDistance.cs" />
+    <Compile Include="TSNE\Distances\DistanceBase.cs" />
+    <Compile Include="TSNE\Distances\EuclideanDistance.cs" />
+    <Compile Include="TSNE\Distances\IndexedItemDistance.cs" />
+    <Compile Include="TSNE\Distances\ManhattanDistance.cs" />
+    <Compile Include="TSNE\Distances\WeightedEuclideanDistance.cs" />
+    <Compile Include="TSNE\Distances\IDistance.cs" />
+    <Compile Include="TSNE\PriorityQueue.cs" />
+    <Compile Include="TSNE\SpacePartitioningTree.cs" />
+    <Compile Include="TSNE\TSNEAlgorithm.cs" />
+    <Compile Include="TSNE\TSNEStatic.cs" />
+    <Compile Include="TSNE\TSNEUtils.cs" />
+    <Compile Include="TSNE\VantagePointTree.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\HeuristicLab.Algorithms.GradientDescent\3.3\HeuristicLab.Algorithms.GradientDescent-3.3.csproj">
+      <Project>{1256B945-EEA9-4BE4-9880-76B5B113F089}</Project>
+      <Name>HeuristicLab.Algorithms.GradientDescent-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm\3.3\HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm-3.3.csproj">
+      <Project>{F409DD9E-1E9C-4EB1-AA3A-9F6E987C6E58}</Project>
+      <Name>HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm-3.3</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Analysis\3.3\HeuristicLab.Analysis-3.3.csproj">
+      <Project>{887425B4-4348-49ED-A457-B7D2C26DDBF9}</Project>
+      <Name>HeuristicLab.Analysis-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Collections\3.3\HeuristicLab.Collections-3.3.csproj">
+      <Project>{958B43BC-CC5C-4FA2-8628-2B3B01D890B6}</Project>
+      <Name>HeuristicLab.Collections-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Common.Resources\3.3\HeuristicLab.Common.Resources-3.3.csproj">
+      <Project>{0E27A536-1C4A-4624-A65E-DC4F4F23E3E1}</Project>
+      <Name>HeuristicLab.Common.Resources-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Common\3.3\HeuristicLab.Common-3.3.csproj">
+      <Project>{A9AD58B9-3EF9-4CC1-97E5-8D909039FF5C}</Project>
+      <Name>HeuristicLab.Common-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Core\3.3\HeuristicLab.Core-3.3.csproj">
+      <Project>{C36BD924-A541-4A00-AFA8-41701378DDC5}</Project>
+      <Name>HeuristicLab.Core-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Data\3.3\HeuristicLab.Data-3.3.csproj">
+      <Project>{BBAB9DF5-5EF3-4BA8-ADE9-B36E82114937}</Project>
+      <Name>HeuristicLab.Data-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Encodings.RealVectorEncoding\3.3\HeuristicLab.Encodings.RealVectorEncoding-3.3.csproj">
+      <Project>{BB6D334A-4BB6-4674-9883-31A6EBB32CAB}</Project>
+      <Name>HeuristicLab.Encodings.RealVectorEncoding-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Encodings.SymbolicExpressionTreeEncoding\3.4\HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4.csproj">
+      <Project>{06D4A186-9319-48A0-BADE-A2058D462EEA}</Project>
+      <Name>HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Operators\3.3\HeuristicLab.Operators-3.3.csproj">
+      <Project>{23DA7FF4-D5B8-41B6-AA96-F0561D24F3EE}</Project>
+      <Name>HeuristicLab.Operators-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Optimization\3.3\HeuristicLab.Optimization-3.3.csproj">
+      <Project>{14AB8D24-25BC-400C-A846-4627AA945192}</Project>
+      <Name>HeuristicLab.Optimization-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Parameters\3.3\HeuristicLab.Parameters-3.3.csproj">
+      <Project>{56F9106A-079F-4C61-92F6-86A84C2D84B7}</Project>
+      <Name>HeuristicLab.Parameters-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Persistence\3.3\HeuristicLab.Persistence-3.3.csproj">
+      <Project>{102BC7D3-0EF9-439C-8F6D-96FF0FDB8E1B}</Project>
+      <Name>HeuristicLab.Persistence-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.PluginInfrastructure\3.3\HeuristicLab.PluginInfrastructure-3.3.csproj">
+      <Project>{94186A6A-5176-4402-AE83-886557B53CCA}</Project>
+      <Name>HeuristicLab.PluginInfrastructure-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.DataAnalysis.Symbolic.Classification\3.4\HeuristicLab.Problems.DataAnalysis.Symbolic.Classification-3.4.csproj">
+      <Project>{05BAE4E1-A9FA-4644-AA77-42558720159E}</Project>
+      <Name>HeuristicLab.Problems.DataAnalysis.Symbolic.Classification-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.DataAnalysis.Symbolic.Regression\3.4\HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4.csproj">
+      <Project>{5AC82412-911B-4FA2-A013-EDC5E3F3FCC2}</Project>
+      <Name>HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.DataAnalysis.Symbolic.TimeSeriesPrognosis\3.4\HeuristicLab.Problems.DataAnalysis.Symbolic.TimeSeriesPrognosis-3.4.csproj">
+      <Project>{07486E68-1517-4B9D-A58D-A38E99AE71AB}</Project>
+      <Name>HeuristicLab.Problems.DataAnalysis.Symbolic.TimeSeriesPrognosis-3.4</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.DataAnalysis.Symbolic\3.4\HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj">
+      <Project>{3D28463F-EC96-4D82-AFEE-38BE91A0CA00}</Project>
+      <Name>HeuristicLab.Problems.DataAnalysis.Symbolic-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.DataAnalysis\3.4\HeuristicLab.Problems.DataAnalysis-3.4.csproj">
+      <Project>{DF87C13E-A889-46FF-8153-66DCAA8C5674}</Project>
+      <Name>HeuristicLab.Problems.DataAnalysis-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.Instances\3.3\HeuristicLab.Problems.Instances-3.3.csproj">
+      <Project>{3540E29E-4793-49E7-8EE2-FEA7F61C3994}</Project>
+      <Name>HeuristicLab.Problems.Instances-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Random\3.3\HeuristicLab.Random-3.3.csproj">
+      <Project>{F4539FB6-4708-40C9-BE64-0A1390AEA197}</Project>
+      <Name>HeuristicLab.Random-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Selection\3.3\HeuristicLab.Selection-3.3.csproj">
+      <Project>{2C36CD4F-E5F5-43A4-801A-201EA895FE17}</Project>
+      <Name>HeuristicLab.Selection-3.3</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="HeuristicLab.snk" />
+    <None Include="Plugin.cs.frame" />
+    <None Include="Properties\AssemblyInfo.cs.frame" />
+  </ItemGroup>
+  <ItemGroup>
+    <BootstrapperPackage Include="Microsoft.Net.Client.3.5">
+      <Visible>False</Visible>
+      <ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
+      <Install>false</Install>
+    </BootstrapperPackage>
+    <BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
+      <Visible>False</Visible>
+      <ProductName>.NET Framework 3.5 SP1</ProductName>
+      <Install>true</Install>
+    </BootstrapperPackage>
+    <BootstrapperPackage Include="Microsoft.Windows.Installer.3.1">
+      <Visible>False</Visible>
+      <ProductName>Windows Installer 3.1</ProductName>
+      <Install>true</Install>
+    </BootstrapperPackage>
+  </ItemGroup>
+  <ItemGroup>
+    <Reference Include="HEAL.Attic, Version=1.0.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\bin\HEAL.Attic.dll</HintPath>
+      <Private>False</Private>
+    </Reference>
+  </ItemGroup>
+  <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+  <PropertyGroup>
+    <PreBuildEvent Condition=" '$(OS)' == 'Windows_NT' ">set Path=%25Path%25;$(ProjectDir);$(SolutionDir)
+set ProjectDir=$(ProjectDir)
+set SolutionDir=$(SolutionDir)
+set Outdir=$(Outdir)
+
+call PreBuildEvent.cmd
+</PreBuildEvent>
+    <PreBuildEvent Condition=" '$(OS)' != 'Windows_NT' ">
+export ProjectDir=$(ProjectDir)
+export SolutionDir=$(SolutionDir)
+
+$SolutionDir/PreBuildEvent.sh
+</PreBuildEvent>
+  </PropertyGroup>
+</Project>
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IDataAnalysisAlgorithm.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IDataAnalysisAlgorithm.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IDataAnalysisAlgorithm.cs	(revision 17041)
@@ -0,0 +1,34 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("76554c78-e43e-4988-910b-0f31338dab47")]
+  /// <summary>
+  /// Interface to represent a data analysis algorithm.
+  /// </summary>
+  public interface IDataAnalysisAlgorithm<T> : IAlgorithm where T : class, IDataAnalysisProblem {
+    new T Problem { get; set; }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessClassificationModelCreator.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessClassificationModelCreator.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessClassificationModelCreator.cs	(revision 17041)
@@ -0,0 +1,34 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("14d873cf-cdfa-40db-80d2-bc25ccf38344")]
+  /// <summary>
+  /// Interface to represent a Gaussian process model creator for classification
+  /// </summary>
+  public interface IGaussianProcessClassificationModelCreator : IGaussianProcessModelCreator {
+    ILookupParameter<IClassificationProblemData> ProblemDataParameter { get; }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModel.cs	(revision 17041)
@@ -0,0 +1,40 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("dc76e136-7a3e-49cd-95b4-bbd3ead06d0c")]
+  /// <summary>
+  /// Interface to represent a Gaussian process posterior
+  /// </summary>
+  public interface IGaussianProcessModel : IConfidenceRegressionModel {
+    double NegativeLogLikelihood { get; }
+    double LooCvNegativeLogPseudoLikelihood { get; }
+    double SigmaNoise { get; }
+    IMeanFunction MeanFunction { get; }
+    ICovarianceFunction CovarianceFunction { get; }
+    double[] HyperparameterGradients { get; }
+
+    void FixParameters();
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModelCreator.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModelCreator.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModelCreator.cs	(revision 17041)
@@ -0,0 +1,41 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.RealVectorEncoding;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("a83cbf4d-6e7c-4e30-904e-d0f0c89d2ed3")]
+  /// <summary>
+  /// Interface to represent a Gaussian process model creator (either regression or classification)
+  /// </summary>
+  public interface IGaussianProcessModelCreator : IOperator {
+    ILookupParameter<RealVector> HyperparameterParameter { get; }
+    ILookupParameter<IMeanFunction> MeanFunctionParameter { get; }
+    ILookupParameter<ICovarianceFunction> CovarianceFunctionParameter { get; }
+    ILookupParameter<IGaussianProcessModel> ModelParameter { get; }
+    ILookupParameter<RealVector> HyperparameterGradientsParameter { get; }
+    ILookupParameter<DoubleValue> NegativeLogLikelihoodParameter { get; }
+    ILookupParameter<DoubleValue> NegativeLogPseudoLikelihoodParameter { get; }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessRegressionModelCreator.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessRegressionModelCreator.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessRegressionModelCreator.cs	(revision 17041)
@@ -0,0 +1,34 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("11e34d2f-d302-4048-b125-7886cf655d6f")]
+  /// <summary>
+  /// Interface to represent a Gaussian process model creator for regression
+  /// </summary>
+  public interface IGaussianProcessRegressionModelCreator : IGaussianProcessModelCreator {
+    ILookupParameter<IRegressionProblemData> ProblemDataParameter { get; }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessSolution.cs	(revision 17041)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("f9bedb56-c034-4bb3-8125-d1146b03376c")]
+  /// <summary>
+  /// Interface to represent a Gaussian process solution (either regression or classification)
+  /// </summary>
+  public interface IGaussianProcessSolution : IConfidenceRegressionSolution {
+    new IGaussianProcessModel Model { get; }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaClassificationSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaClassificationSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaClassificationSolution.cs	(revision 17041)
@@ -0,0 +1,30 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("afb3d86e-f5a9-4bd5-a5a5-35337c1163e9")]
+  public interface INcaClassificationSolution : IClassificationSolution {
+    new INcaModel Model { get; }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaModel.cs	(revision 17041)
@@ -0,0 +1,33 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("52f233e3-90de-4de9-bd8f-eca1b9eab1ac")]
+  public interface INcaModel : IClassificationModel {
+    new INcaClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData);
+
+    double[,] Reduce(IDataset dataset, IEnumerable<int> rows);
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourClassificationSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourClassificationSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourClassificationSolution.cs	(revision 17041)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("ad083c00-f5fb-4b79-a36f-489523f98be9")]
+  /// <summary>
+  /// Interface to represent a nearest neighbour classification solution
+  /// </summary>
+  public interface INearestNeighbourClassificationSolution : IClassificationSolution {
+    new INearestNeighbourModel Model { get; }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourModel.cs	(revision 17041)
@@ -0,0 +1,32 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("ceda9f55-7424-4f9a-9961-db300d093f96")]
+  /// <summary>
+  /// Interface to represent a nearest neighbour model for either regression or classification
+  /// </summary>
+  public interface INearestNeighbourModel : IRegressionModel, IClassificationModel {
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourRegressionSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourRegressionSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourRegressionSolution.cs	(revision 17041)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("ce11dac3-5b79-4390-80cc-dddeb9d0250c")]
+  /// <summary>
+  /// Interface to represent a nearest neighbour regression solution
+  /// </summary>
+  public interface INearestNeighbourRegressionSolution : IRegressionSolution {
+    new INearestNeighbourModel Model { get; }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkClassificationSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkClassificationSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkClassificationSolution.cs	(revision 17041)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("5c569264-6b1d-4a7c-8c3a-938576f07c6f")]
+  /// <summary>
+  /// Interface to represent a neural network classification solution
+  /// </summary>
+  public interface INeuralNetworkClassificationSolution : IClassificationSolution {
+    new INeuralNetworkModel Model { get; }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleClassificationSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleClassificationSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleClassificationSolution.cs	(revision 17041)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("d9c3883e-99bb-4730-b36c-18ed2a52cbe3")]
+  /// <summary>
+  /// Interface to represent a neural network ensemble classification solution
+  /// </summary>
+  public interface INeuralNetworkEnsembleClassificationSolution : IClassificationSolution {
+    new INeuralNetworkEnsembleModel Model { get; }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleModel.cs	(revision 17041)
@@ -0,0 +1,32 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("8cc4503e-dec8-43d9-a966-3619ace0da41")]
+  /// <summary>
+  /// Interface to represent a neural network ensemble model for either regression or classification
+  /// </summary>
+  public interface INeuralNetworkEnsembleModel : IRegressionModel, IClassificationModel {
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleRegressionSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleRegressionSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleRegressionSolution.cs	(revision 17041)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("9cbae9de-c128-45d8-9013-4ac70f641b08")]
+  /// <summary>
+  /// Interface to represent a neural network ensemble regression solution
+  /// </summary>
+  public interface INeuralNetworkEnsembleRegressionSolution : IRegressionSolution {
+    new INeuralNetworkEnsembleModel Model { get; }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkModel.cs	(revision 17041)
@@ -0,0 +1,32 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("21800d09-cdf8-4faf-a832-0a709ab84f33")]
+  /// <summary>
+  /// Interface to represent a neural network model for either regression or classification
+  /// </summary>
+  public interface INeuralNetworkModel : IRegressionModel, IClassificationModel {
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkRegressionSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkRegressionSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkRegressionSolution.cs	(revision 17041)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("f5a0df9f-1148-40d8-a13d-f5e114c2c95f")]
+  /// <summary>
+  /// Interface to represent a neural network regression solution
+  /// </summary>
+  public interface INeuralNetworkRegressionSolution : IRegressionSolution {
+    new INeuralNetworkModel Model { get; }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestClassificationSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestClassificationSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestClassificationSolution.cs	(revision 17041)
@@ -0,0 +1,34 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("ff6d9203-3685-418b-b6cc-2312b50c4014")]
+  /// <summary>
+  /// Interface to represent a random forest classification solution
+  /// </summary>
+  public interface IRandomForestClassificationSolution : IClassificationSolution {
+    new IRandomForestModel Model { get; }
+    int NumberOfTrees { get; }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestModel.cs	(revision 17041)
@@ -0,0 +1,35 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("627fb9cf-b3fe-4f9b-a76b-f0f1b7a31f0c")]
+  /// <summary>
+  /// Interface to represent a random forest model for either regression or classification
+  /// </summary>
+  public interface IRandomForestModel : IConfidenceRegressionModel, IClassificationModel {
+    int NumberOfTrees { get; }
+    ISymbolicExpressionTree ExtractTree(int treeIdx); // returns a specific tree from the random forest as a ISymbolicRegressionModel
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestRegressionSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestRegressionSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestRegressionSolution.cs	(revision 17041)
@@ -0,0 +1,34 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("17e636cc-fa01-4844-8e48-bd7d525eac3a")]
+  /// <summary>
+  /// Interface to represent a random forest regression solution
+  /// </summary>
+  public interface IRandomForestRegressionSolution : IConfidenceRegressionSolution {
+    new IRandomForestModel Model { get; }
+    int NumberOfTrees { get; }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineModel.cs	(revision 17041)
@@ -0,0 +1,36 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+using LibSVM;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("5284987f-4f3e-43dd-9bc8-a8b3c7d952ce")]
+  /// <summary>
+  /// Interface to represent a support vector machine model
+  /// </summary>
+  public interface ISupportVectorMachineModel : IDataAnalysisModel, IRegressionModel, IClassificationModel {
+    svm_model Model { get; }
+    RangeTransform RangeTransform { get; }
+    Dataset SupportVectors { get; }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineSolution.cs	(revision 17041)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("598d2904-58e5-4a2f-8cd4-378ed3fa6132")]
+  /// <summary>
+  /// Interface to represent a support vector solution (either regression or classification)
+  /// </summary>
+  public interface ISupportVectorMachineSolution : IDataAnalysisSolution {
+    new ISupportVectorMachineModel Model { get; }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/CicularKernel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/CicularKernel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/CicularKernel.cs	(revision 17041)
@@ -0,0 +1,61 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("BB5992FF-783A-490E-91FE-C0782BD1CBB9")]
+  [Item("CircularKernel", "A circular kernel function 2*pi*(acos(-d)-d*(1-d²)^(0.5)) where n = ||x-c|| and d = n/beta \n  As described in http://crsouza.com/2010/03/17/kernel-functions-for-machine-learning-applications/")]
+  public class CircularKernel : KernelBase {
+    [StorableConstructor]
+    protected CircularKernel(StorableConstructorFlag _) : base(_) { }
+
+    protected CircularKernel(CircularKernel original, Cloner cloner) : base(original, cloner) { }
+
+    public CircularKernel() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CircularKernel(this, cloner);
+    }
+
+    protected override double Get(double norm) {
+      if (Beta == null) throw new InvalidOperationException("Can not calculate kernel distance while Beta is null");
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      if (norm >= beta) return 0;
+      var d = norm / beta;
+      return 2 * Math.PI * (Math.Acos(-d) - d * Math.Sqrt(1 - d * d));
+    }
+
+    // 4*pi*n^3 / (beta^4 * sqrt(1-n^2/beta^2) 
+    protected override double GetGradient(double norm) {
+      if (Beta == null) throw new InvalidOperationException("Can not calculate kernel distance gradient while Beta is null");
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      if (beta < norm) return 0;
+      var d = norm / beta;
+      return -4 * Math.PI * d * d * d / beta * Math.Sqrt(1 - d * d);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/GaussianKernel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/GaussianKernel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/GaussianKernel.cs	(revision 17041)
@@ -0,0 +1,62 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("CFF2B805-FE21-427B-B899-531D3AB1C7EF")]
+  [Item("GaussianKernel", "A kernel function that uses Gaussian function exp(-n²/beta²). As described in http://crsouza.com/2010/03/17/kernel-functions-for-machine-learning-applications/")]
+  public class GaussianKernel : KernelBase {
+    [StorableConstructor]
+    protected GaussianKernel(StorableConstructorFlag _) : base(_) { }
+
+    protected GaussianKernel(GaussianKernel original, Cloner cloner) : base(original, cloner) { }
+
+    public GaussianKernel() {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianKernel(this, cloner);
+    }
+
+    protected override double Get(double norm) {
+      if (Beta == null) throw new InvalidOperationException("Can not calculate kernel distance while Beta is null");
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      var d = norm / beta;
+      return Math.Exp(-d * d);
+    }
+
+    //2 * n²/b²* 1/b * exp(-n²/b²)
+    protected override double GetGradient(double norm) {
+      if (Beta == null) throw new InvalidOperationException("Can not calculate kernel distance gradient while Beta is null");
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      var d = norm / beta;
+      return 2 * d * d / beta * Math.Exp(-d * d);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/IKernel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/IKernel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/IKernel.cs	(revision 17041)
@@ -0,0 +1,35 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+
+using System;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("adb61694-6937-4c93-aa8d-902b54811e37")]
+  public interface IKernel : ICovarianceFunction {
+    double? Beta { get; set; } // a kernel parameter
+    IDistance Distance { get; set; } // the distance function to use
+
+    event EventHandler BetaChanged;
+    event EventHandler DistanceChanged;
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/InverseMultiquadraticKernel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/InverseMultiquadraticKernel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/InverseMultiquadraticKernel.cs	(revision 17041)
@@ -0,0 +1,61 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("891AC2D7-520C-4D34-9CBF-174DFA290C12")]
+  [Item("InverseMultiquadraticKernel", "A kernel function that uses the inverse multi-quadratic function  1 / sqrt(1+||x-c||²/beta²). Similar to http://crsouza.com/2010/03/17/kernel-functions-for-machine-learning-applications/ with beta as a scaling factor.")]
+  public class InverseMultiquadraticKernel : KernelBase {
+    private const double C = 1.0;
+
+    [StorableConstructor]
+    protected InverseMultiquadraticKernel(StorableConstructorFlag _) : base(_) { }
+
+    protected InverseMultiquadraticKernel(InverseMultiquadraticKernel original, Cloner cloner) : base(original, cloner) { }
+
+    public InverseMultiquadraticKernel() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new InverseMultiquadraticKernel(this, cloner);
+    }
+
+    protected override double Get(double norm) {
+      if (Beta == null) throw new InvalidOperationException("Can not calculate kernel distance while Beta is null");
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      var d = norm / beta;
+      return 1 / Math.Sqrt(C + d * d);
+    }
+
+    //n²/(b³(n²/b² + C)^1.5)
+    protected override double GetGradient(double norm) {
+      if (Beta == null) throw new InvalidOperationException("Can not calculate kernel distance gradient while Beta is null");
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      var d = norm / beta;
+      return d * d / (beta * Math.Pow(d * d + C, 1.5));
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/KernelBase.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/KernelBase.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/KernelBase.cs	(revision 17041)
@@ -0,0 +1,136 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("3449B830-E1E5-4176-B56D-AA32235F061B")]
+  public abstract class KernelBase : ParameterizedNamedItem, IKernel {
+
+    private const string DistanceParameterName = "Distance";
+
+    public IValueParameter<IDistance> DistanceParameter {
+      get { return (IValueParameter<IDistance>)Parameters[DistanceParameterName]; }
+    }
+
+    [Storable]
+    private double? beta;
+    public double? Beta {
+      get { return beta; }
+      set {
+        if (value != beta) {
+          beta = value;
+          RaiseBetaChanged();
+        }
+      }
+    }
+
+    public IDistance Distance {
+      get { return DistanceParameter.Value; }
+      set {
+        if (DistanceParameter.Value != value) {
+          DistanceParameter.Value = value;
+        }
+      }
+    }
+
+    [StorableConstructor]
+    protected KernelBase(StorableConstructorFlag _) : base(_) { }
+
+    protected KernelBase(KernelBase original, Cloner cloner)
+      : base(original, cloner) {
+      beta = original.beta;
+      RegisterEvents();
+    }
+
+    protected KernelBase() {
+      Parameters.Add(new ValueParameter<IDistance>(DistanceParameterName, "The distance function used for kernel calculation"));
+      DistanceParameter.Value = new EuclideanDistance();
+      RegisterEvents();
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      RegisterEvents();
+    }
+
+    private void RegisterEvents() {
+      DistanceParameter.ValueChanged += (sender, args) => RaiseDistanceChanged();
+    }
+
+    public double Get(object a, object b) {
+      return Get(Distance.Get(a, b));
+    }
+
+    protected abstract double Get(double norm);
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return Beta.HasValue ? 0 : 1;
+    }
+
+    public void SetParameter(double[] p) {
+      if (p != null && p.Length == 1) Beta = new double?(p[0]);
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+      if (p.Length != GetNumberOfParameters(columnIndices.Length)) throw new ArgumentException("Illegal parametrization");
+      var myClone = (KernelBase)Clone();
+      myClone.SetParameter(p);
+      var cov = new ParameterizedCovarianceFunction {
+        Covariance = (x, i, j) => myClone.Get(GetNorm(x, x, i, j, columnIndices)),
+        CrossCovariance = (x, xt, i, j) => myClone.Get(GetNorm(x, xt, i, j, columnIndices)),
+        CovarianceGradient = (x, i, j) => new List<double> { myClone.GetGradient(GetNorm(x, x, i, j, columnIndices)) }
+      };
+      return cov;
+    }
+
+    protected abstract double GetGradient(double norm);
+
+    protected double GetNorm(double[,] x, double[,] xt, int i, int j, int[] columnIndices) {
+      var dist = Distance as IDistance<IEnumerable<double>>;
+      if (dist == null) throw new ArgumentException("The distance needs to apply to double vectors");
+      var r1 = columnIndices.Select(c => x[i, c]);
+      var r2 = columnIndices.Select(c => xt[j, c]);
+      return dist.Get(r1, r2);
+    }
+
+    #region events
+    public event EventHandler BetaChanged;
+    public event EventHandler DistanceChanged;
+
+    protected void RaiseBetaChanged() {
+      var handler = BetaChanged;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
+
+    protected void RaiseDistanceChanged() {
+      var handler = DistanceChanged;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/MultiquadraticKernel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/MultiquadraticKernel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/MultiquadraticKernel.cs	(revision 17041)
@@ -0,0 +1,63 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("9017554C-BB2A-45E1-9050-1260CB98D04A")]
+  // conditionally positive definite. (need to add polynomials) see http://num.math.uni-goettingen.de/schaback/teaching/sc.pdf 
+  [Item("MultiquadraticKernel", "A kernel function that uses the multi-quadratic function sqrt(1+||x-c||²/beta²). Similar to http://crsouza.com/2010/03/17/kernel-functions-for-machine-learning-applications/ with beta as a scaling factor.")]
+  public class MultiquadraticKernel : KernelBase {
+
+    private const double C = 1.0;
+
+    [StorableConstructor]
+    protected MultiquadraticKernel(StorableConstructorFlag _) : base(_) { }
+
+    protected MultiquadraticKernel(MultiquadraticKernel original, Cloner cloner) : base(original, cloner) { }
+
+    public MultiquadraticKernel() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MultiquadraticKernel(this, cloner);
+    }
+
+    protected override double Get(double norm) {
+      if (Beta == null) throw new InvalidOperationException("Can not calculate kernel distance while Beta is null");
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      var d = norm / beta;
+      return Math.Sqrt(C + d * d);
+    }
+
+    //-n²/(d³*sqrt(C+n²/d²))
+    protected override double GetGradient(double norm) {
+      if (Beta == null) throw new InvalidOperationException("Can not calculate kernel distance gradient while Beta is null");
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      var d = norm / beta;
+      return -d * d / (beta * Math.Sqrt(C + d * d));
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/PolysplineKernel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/PolysplineKernel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/PolysplineKernel.cs	(revision 17041)
@@ -0,0 +1,75 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("0618DE2C-6A69-4086-ADE1-F983AC626F42")]
+  // conditionally positive definite. (need to add polynomials) see http://num.math.uni-goettingen.de/schaback/teaching/sc.pdf 
+  [Item("PolysplineKernel", "A kernel function that uses the polyharmonic function (||x-c||/Beta)^Degree as given in http://num.math.uni-goettingen.de/schaback/teaching/sc.pdf with beta as a scaling parameters.")]
+  public class PolysplineKernel : KernelBase {
+
+    private const string DegreeParameterName = "Degree";
+
+    public IFixedValueParameter<DoubleValue> DegreeParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[DegreeParameterName]; }
+    }
+
+    public DoubleValue Degree {
+      get { return DegreeParameter.Value; }
+    }
+
+    [StorableConstructor]
+    protected PolysplineKernel(StorableConstructorFlag _) : base(_) { }
+
+    protected PolysplineKernel(PolysplineKernel original, Cloner cloner) : base(original, cloner) { }
+
+    public PolysplineKernel() {
+      Parameters.Add(new FixedValueParameter<DoubleValue>(DegreeParameterName, "The degree of the kernel. Needs to be greater than zero.", new DoubleValue(1.0)));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new PolysplineKernel(this, cloner);
+    }
+
+    protected override double Get(double norm) {
+      if (Beta == null) throw new InvalidOperationException("Can not calculate kernel distance gradient while Beta is null");
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      var d = norm / beta;
+      return Math.Pow(d, Degree.Value);
+    }
+
+    //-degree/beta * (norm/beta)^degree
+    protected override double GetGradient(double norm) {
+      if (Beta == null) throw new InvalidOperationException("Can not calculate kernel distance gradient while Beta is null");
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      var d = norm / beta;
+      return -Degree.Value / beta * Math.Pow(d, Degree.Value);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/ThinPlatePolysplineKernel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/ThinPlatePolysplineKernel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/ThinPlatePolysplineKernel.cs	(revision 17041)
@@ -0,0 +1,76 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("4D79FB1C-94CD-4450-8CD0-F9594A5F03FE")]
+  // conditionally positive definite. (need to add polynomials) see http://num.math.uni-goettingen.de/schaback/teaching/sc.pdf 
+  [Item("ThinPlatePolysplineKernel", "A kernel function that uses the ThinPlatePolyspline function (||x-c||/Beta)^(Degree)*log(||x-c||/Beta) as described in \"Thin-Plate Spline Radial Basis Function Scheme for Advection-Diffusion Problems\" with beta as a scaling parameter.")]
+  public class ThinPlatePolysplineKernel : KernelBase {
+
+    private const string DegreeParameterName = "Degree";
+
+    public IFixedValueParameter<DoubleValue> DegreeParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[DegreeParameterName]; }
+    }
+    public DoubleValue Degree {
+      get { return DegreeParameter.Value; }
+    }
+
+    [StorableConstructor]
+    protected ThinPlatePolysplineKernel(StorableConstructorFlag _) : base(_) { }
+
+    protected ThinPlatePolysplineKernel(ThinPlatePolysplineKernel original, Cloner cloner) : base(original, cloner) { }
+
+    public ThinPlatePolysplineKernel() {
+      Parameters.Add(new FixedValueParameter<DoubleValue>(DegreeParameterName, "The degree of the kernel. Needs to be greater than zero.", new DoubleValue(2.0)));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new ThinPlatePolysplineKernel(this, cloner);
+    }
+
+    protected override double Get(double norm) {
+      if (Beta == null) throw new InvalidOperationException("Can not calculate kernel distance while Beta is null");
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      var d = norm / beta;
+      if (Math.Abs(d) < double.Epsilon) return 0;
+      return Math.Pow(d, Degree.Value) * Math.Log(d);
+    }
+
+    // (Degree/beta) * (norm/beta)^Degree * log(norm/beta) 
+    protected override double GetGradient(double norm) {
+      if (Beta == null) throw new InvalidOperationException("Can not calculate kernel distance gradient while Beta is null");
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      var d = norm / beta;
+      if (Math.Abs(d) < double.Epsilon) return 0;
+      return Degree.Value / beta * Math.Pow(d, Degree.Value) * Math.Log(d);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegression.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegression.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegression.cs	(revision 17041)
@@ -0,0 +1,141 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.PluginInfrastructure;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("Kernel Ridge Regression", "Kernelized ridge regression e.g. for radial basis function (RBF) regression.")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 100)]
+  [StorableType("8AD45266-68CA-4710-A99C-59952132AA9D")]
+  public sealed class KernelRidgeRegression : BasicAlgorithm, IDataAnalysisAlgorithm<IRegressionProblem> {
+    private const string SolutionResultName = "Kernel ridge regression solution";
+
+    public override bool SupportsPause {
+      get { return false; }
+    }
+    public override Type ProblemType {
+      get { return typeof(IRegressionProblem); }
+    }
+    public new IRegressionProblem Problem {
+      get { return (IRegressionProblem)base.Problem; }
+      set { base.Problem = value; }
+    }
+
+    #region parameter names
+    private const string KernelParameterName = "Kernel";
+    private const string ScaleInputVariablesParameterName = "ScaleInputVariables";
+    private const string LambdaParameterName = "LogLambda";
+    private const string BetaParameterName = "Beta";
+    #endregion
+
+    #region parameter properties
+    public IConstrainedValueParameter<IKernel> KernelParameter {
+      get { return (IConstrainedValueParameter<IKernel>)Parameters[KernelParameterName]; }
+    }
+
+    public IFixedValueParameter<BoolValue> ScaleInputVariablesParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[ScaleInputVariablesParameterName]; }
+    }
+
+    public IFixedValueParameter<DoubleValue> LogLambdaParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[LambdaParameterName]; }
+    }
+
+    public IFixedValueParameter<DoubleValue> BetaParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[BetaParameterName]; }
+    }
+    #endregion
+
+    #region properties
+    public IKernel Kernel {
+      get { return KernelParameter.Value; }
+    }
+
+    public bool ScaleInputVariables {
+      get { return ScaleInputVariablesParameter.Value.Value; }
+      set { ScaleInputVariablesParameter.Value.Value = value; }
+    }
+
+    public double LogLambda {
+      get { return LogLambdaParameter.Value.Value; }
+      set { LogLambdaParameter.Value.Value = value; }
+    }
+
+    public double Beta {
+      get { return BetaParameter.Value.Value; }
+      set { BetaParameter.Value.Value = value; }
+    }
+    #endregion
+
+    [StorableConstructor]
+    private KernelRidgeRegression(StorableConstructorFlag _) : base(_) { }
+    private KernelRidgeRegression(KernelRidgeRegression original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public KernelRidgeRegression() {
+      Problem = new RegressionProblem();
+      var values = new ItemSet<IKernel>(ApplicationManager.Manager.GetInstances<IKernel>());
+      Parameters.Add(new ConstrainedValueParameter<IKernel>(KernelParameterName, "The kernel", values, values.OfType<GaussianKernel>().FirstOrDefault()));
+      Parameters.Add(new FixedValueParameter<BoolValue>(ScaleInputVariablesParameterName, "Set to true if the input variables should be scaled to the interval [0..1]", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(LambdaParameterName, "The log10-transformed weight for the regularization term lambda [-inf..+inf]. Small values produce more complex models, large values produce models with larger errors. Set to very small value (e.g. -1.0e15) for almost exact approximation", new DoubleValue(-2)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(BetaParameterName, "The inverse width of the kernel ]0..+inf]. The distance between points is divided by this value before being plugged into the kernel.", new DoubleValue(2)));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new KernelRidgeRegression(this, cloner);
+    }
+
+    protected override void Run(CancellationToken cancellationToken) {
+      double rmsError, looCvRMSE;
+      var kernel = Kernel;
+      kernel.Beta = Beta;
+      var solution = CreateRadialBasisRegressionSolution(Problem.ProblemData, kernel, Math.Pow(10, LogLambda), ScaleInputVariables, out rmsError, out looCvRMSE);
+      Results.Add(new Result(SolutionResultName, "The kernel ridge regression solution.", solution));
+      Results.Add(new Result("RMSE (test)", "The root mean squared error of the solution on the test set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("RMSE (LOO-CV)", "The leave-one-out-cross-validation root mean squared error", new DoubleValue(looCvRMSE)));
+    }
+
+    public static IRegressionSolution CreateRadialBasisRegressionSolution(IRegressionProblemData problemData, ICovarianceFunction kernel, double lambda, bool scaleInputs, out double rmsError, out double looCvRMSE) {
+      var model = KernelRidgeRegressionModel.Create(problemData.Dataset, problemData.TargetVariable, problemData.AllowedInputVariables, problemData.TrainingIndices, scaleInputs, kernel, lambda);
+      rmsError = double.NaN;
+      if (problemData.TestIndices.Any()) {
+        rmsError = Math.Sqrt(model.GetEstimatedValues(problemData.Dataset, problemData.TestIndices)
+          .Zip(problemData.TargetVariableTestValues, (a, b) => (a - b) * (a - b))
+          .Average());
+      }
+      var solution = model.CreateRegressionSolution((IRegressionProblemData)problemData.Clone());
+      solution.Model.Name = "Kernel ridge regression model";
+      solution.Name = SolutionResultName;
+      looCvRMSE = model.LooCvRMSE;
+      return solution;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegressionModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegressionModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegressionModel.cs	(revision 17041)
@@ -0,0 +1,238 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("4148D88C-6081-4D84-B718-C949CA5AA766")]
+  [Item("KernelRidgeRegressionModel", "A kernel ridge regression model")]
+  public sealed class KernelRidgeRegressionModel : RegressionModel {
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return allowedInputVariables; }
+    }
+
+    [Storable]
+    private readonly string[] allowedInputVariables;
+    public string[] AllowedInputVariables {
+      get { return allowedInputVariables.ToArray(); }
+    }
+
+
+    [Storable]
+    public double LooCvRMSE { get; private set; }
+
+    [Storable]
+    private readonly double[] alpha;
+
+    [Storable]
+    private readonly double[,] trainX; // it is better to store the original training dataset completely because this is more efficient in persistence
+
+    [Storable]
+    private readonly ITransformation<double>[] scaling;
+
+    [Storable]
+    private readonly ICovarianceFunction kernel;
+
+    [Storable]
+    private readonly double lambda;
+
+    [Storable]
+    private readonly double yOffset; // implementation works for zero-mean, unit-variance target variables
+
+    [Storable]
+    private readonly double yScale;
+
+    [StorableConstructor]
+    private KernelRidgeRegressionModel(StorableConstructorFlag _) : base(_) { }
+    private KernelRidgeRegressionModel(KernelRidgeRegressionModel original, Cloner cloner)
+      : base(original, cloner) {
+      // shallow copies of arrays because they cannot be modified
+      allowedInputVariables = original.allowedInputVariables;
+      alpha = original.alpha;
+      trainX = original.trainX;
+      scaling = original.scaling;
+      lambda = original.lambda;
+      LooCvRMSE = original.LooCvRMSE;
+
+      yOffset = original.yOffset;
+      yScale = original.yScale;
+      kernel = original.kernel;
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new KernelRidgeRegressionModel(this, cloner);
+    }
+
+    public static KernelRidgeRegressionModel Create(IDataset dataset, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<int> rows,
+      bool scaleInputs, ICovarianceFunction kernel, double lambda = 0.1) {
+      var trainingRows = rows.ToArray();
+      var model = new KernelRidgeRegressionModel(dataset, targetVariable, allowedInputVariables, trainingRows, scaleInputs, kernel, lambda);
+
+      try {
+        int info;
+        int n = model.trainX.GetLength(0);
+        alglib.densesolverreport denseSolveRep;
+        var gram = BuildGramMatrix(model.trainX, lambda, kernel);
+        var l = new double[n, n];
+        Array.Copy(gram, l, l.Length);
+
+        double[] alpha = new double[n];
+        double[,] invG;
+        var y = dataset.GetDoubleValues(targetVariable, trainingRows).ToArray();
+        for (int i = 0; i < y.Length; i++) {
+          y[i] -= model.yOffset;
+          y[i] *= model.yScale;
+        }
+        // cholesky decomposition
+        var res = alglib.trfac.spdmatrixcholesky(ref l, n, false);
+        if (res == false) { //try lua decomposition if cholesky faild
+          int[] pivots;
+          var lua = new double[n, n];
+          Array.Copy(gram, lua, lua.Length);
+          alglib.rmatrixlu(ref lua, n, n, out pivots);
+          alglib.rmatrixlusolve(lua, pivots, n, y, out info, out denseSolveRep, out alpha);
+          if (info != 1) throw new ArgumentException("Could not create model.");
+          alglib.matinvreport rep;
+          invG = lua;  // rename
+          alglib.rmatrixluinverse(ref invG, pivots, n, out info, out rep);
+        } else {
+          alglib.spdmatrixcholeskysolve(l, n, false, y, out info, out denseSolveRep, out alpha);
+          if (info != 1) throw new ArgumentException("Could not create model.");
+          // for LOO-CV we need to build the inverse of the gram matrix
+          alglib.matinvreport rep;
+          invG = l;   // rename 
+          alglib.spdmatrixcholeskyinverse(ref invG, n, false, out info, out rep);
+        }
+        if (info != 1) throw new ArgumentException("Could not invert Gram matrix.");
+
+        var ssqLooError = 0.0;
+        for (int i = 0; i < n; i++) {
+          var pred_i = Util.ScalarProd(Util.GetRow(gram, i).ToArray(), alpha);
+          var looPred_i = pred_i - alpha[i] / invG[i, i];
+          var error = (y[i] - looPred_i) / model.yScale;
+          ssqLooError += error * error;
+        }
+
+        Array.Copy(alpha, model.alpha, n);
+        model.LooCvRMSE = Math.Sqrt(ssqLooError / n);
+      } catch (alglib.alglibexception ae) {
+        // wrap exception so that calling code doesn't have to know about alglib implementation
+        throw new ArgumentException("There was a problem in the calculation of the kernel ridge regression model", ae);
+      }
+      return model;
+    }
+
+    private KernelRidgeRegressionModel(IDataset dataset, string targetVariable, IEnumerable<string> allowedInputVariables, int[] rows,
+      bool scaleInputs, ICovarianceFunction kernel, double lambda = 0.1) : base(targetVariable) {
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+      if (kernel.GetNumberOfParameters(this.allowedInputVariables.Length) > 0) throw new ArgumentException("All parameters in the kernel function must be specified.");
+      name = ItemName;
+      description = ItemDescription;
+
+      this.kernel = (ICovarianceFunction)kernel.Clone();
+      this.lambda = lambda;
+      if (scaleInputs) scaling = CreateScaling(dataset, rows, this.allowedInputVariables);
+      trainX = ExtractData(dataset, rows, this.allowedInputVariables, scaling);
+      var y = dataset.GetDoubleValues(targetVariable, rows).ToArray();
+      yOffset = y.Average();
+      yScale = 1.0 / y.StandardDeviation();
+      alpha = new double[trainX.GetLength(0)];
+    }
+
+
+    #region IRegressionModel Members
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      var newX = ExtractData(dataset, rows, allowedInputVariables, scaling);
+      var dim = newX.GetLength(1);
+      var cov = kernel.GetParameterizedCovarianceFunction(new double[0], Enumerable.Range(0, dim).ToArray());
+
+      var pred = new double[newX.GetLength(0)];
+      for (int i = 0; i < pred.Length; i++) {
+        double sum = 0.0;
+        for (int j = 0; j < alpha.Length; j++) {
+          sum += alpha[j] * cov.CrossCovariance(trainX, newX, j, i);
+        }
+        pred[i] = sum / yScale + yOffset;
+      }
+      return pred;
+    }
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RegressionSolution(this, new RegressionProblemData(problemData));
+    }
+    #endregion
+
+    #region helpers
+    private static double[,] BuildGramMatrix(double[,] data, double lambda, ICovarianceFunction kernel) {
+      var n = data.GetLength(0);
+      var dim = data.GetLength(1);
+      var cov = kernel.GetParameterizedCovarianceFunction(new double[0], Enumerable.Range(0, dim).ToArray());
+      var gram = new double[n, n];
+      // G = (K + λ I) 
+      for (var i = 0; i < n; i++) {
+        for (var j = i; j < n; j++) {
+          gram[i, j] = gram[j, i] = cov.Covariance(data, i, j); // symmetric matrix 
+        }
+        gram[i, i] += lambda;
+      }
+      return gram;
+    }
+
+    private static ITransformation<double>[] CreateScaling(IDataset dataset, int[] rows, IReadOnlyCollection<string> allowedInputVariables) {
+      var trans = new ITransformation<double>[allowedInputVariables.Count];
+      int i = 0;
+      foreach (var variable in allowedInputVariables) {
+        var lin = new LinearTransformation(allowedInputVariables);
+        var max = dataset.GetDoubleValues(variable, rows).Max();
+        var min = dataset.GetDoubleValues(variable, rows).Min();
+        lin.Multiplier = 1.0 / (max - min);
+        lin.Addend = -min / (max - min);
+        trans[i] = lin;
+        i++;
+      }
+      return trans;
+    }
+
+    private static double[,] ExtractData(IDataset dataset, IEnumerable<int> rows, IReadOnlyCollection<string> allowedInputVariables, ITransformation<double>[] scaling = null) {
+      double[][] variables;
+      if (scaling != null) {
+        variables =
+          allowedInputVariables.Select((var, i) => scaling[i].Apply(dataset.GetDoubleValues(var, rows)).ToArray())
+            .ToArray();
+      } else {
+        variables =
+        allowedInputVariables.Select(var => dataset.GetDoubleValues(var, rows).ToArray()).ToArray();
+      }
+      int n = variables.First().Length;
+      var res = new double[n, variables.Length];
+      for (int r = 0; r < n; r++)
+        for (int c = 0; c < variables.Length; c++) {
+          res[r, c] = variables[c][r];
+        }
+      return res;
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearDiscriminantAnalysis.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearDiscriminantAnalysis.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearDiscriminantAnalysis.cs	(revision 17041)
@@ -0,0 +1,116 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Classification;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Linear discriminant analysis classification algorithm.
+  /// </summary>
+  [Item("Linear Discriminant Analysis (LDA)", "Linear discriminant analysis classification algorithm (wrapper for ALGLIB).")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 100)]
+  [StorableType("032E1FDE-D140-47BF-8EB1-D63EC33B0629")]
+  public sealed class LinearDiscriminantAnalysis : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+    private const string LinearDiscriminantAnalysisSolutionResultName = "Linear discriminant analysis solution";
+
+    [StorableConstructor]
+    private LinearDiscriminantAnalysis(StorableConstructorFlag _) : base(_) { }
+    private LinearDiscriminantAnalysis(LinearDiscriminantAnalysis original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public LinearDiscriminantAnalysis()
+      : base() {
+      Problem = new ClassificationProblem();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new LinearDiscriminantAnalysis(this, cloner);
+    }
+
+    #region Fisher LDA
+    protected override void Run(CancellationToken cancellationToken) {
+      var solution = CreateLinearDiscriminantAnalysisSolution(Problem.ProblemData);
+      Results.Add(new Result(LinearDiscriminantAnalysisSolutionResultName, "The linear discriminant analysis.", solution));
+    }
+
+    public static IClassificationSolution CreateLinearDiscriminantAnalysisSolution(IClassificationProblemData problemData) {
+      var dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+      int nClasses = problemData.ClassNames.Count();
+      var doubleVariableNames = allowedInputVariables.Where(dataset.VariableHasType<double>).ToArray();
+      var factorVariableNames = allowedInputVariables.Where(dataset.VariableHasType<string>).ToArray();
+      double[,] inputMatrix = dataset.ToArray(doubleVariableNames.Concat(new string[] { targetVariable }), rows);
+
+      var factorVariables = dataset.GetFactorVariableValues(factorVariableNames, rows);
+      var factorMatrix = dataset.ToArray(factorVariables, rows);
+
+      inputMatrix = factorMatrix.HorzCat(inputMatrix);
+
+      if (inputMatrix.ContainsNanOrInfinity())
+        throw new NotSupportedException("Linear discriminant analysis does not support NaN or infinity values in the input dataset.");
+
+      // change class values into class index
+      int targetVariableColumn = inputMatrix.GetLength(1) - 1;
+      List<double> classValues = problemData.ClassValues.OrderBy(x => x).ToList();
+      for (int row = 0; row < inputMatrix.GetLength(0); row++) {
+        inputMatrix[row, targetVariableColumn] = classValues.IndexOf(inputMatrix[row, targetVariableColumn]);
+      }
+      int info;
+      double[] w;
+      alglib.fisherlda(inputMatrix, inputMatrix.GetLength(0), inputMatrix.GetLength(1) - 1, nClasses, out info, out w);
+      if (info < 1) throw new ArgumentException("Error in calculation of linear discriminant analysis solution");
+
+      var nFactorCoeff = factorMatrix.GetLength(1);
+      var tree = LinearModelToTreeConverter.CreateTree(factorVariables, w.Take(nFactorCoeff).ToArray(),
+        doubleVariableNames, w.Skip(nFactorCoeff).Take(doubleVariableNames.Length).ToArray());
+
+      var model = CreateDiscriminantFunctionModel(tree, new SymbolicDataAnalysisExpressionTreeLinearInterpreter(), problemData, rows);
+      SymbolicDiscriminantFunctionClassificationSolution solution = new SymbolicDiscriminantFunctionClassificationSolution(model, (IClassificationProblemData)problemData.Clone());
+
+      return solution;
+    }
+    #endregion
+
+    private static SymbolicDiscriminantFunctionClassificationModel CreateDiscriminantFunctionModel(ISymbolicExpressionTree tree,
+      ISymbolicDataAnalysisExpressionTreeInterpreter interpreter,
+      IClassificationProblemData problemData,
+      IEnumerable<int> rows) {
+      var model = new SymbolicDiscriminantFunctionClassificationModel(problemData.TargetVariable, tree, interpreter, new AccuracyMaximizationThresholdCalculator());
+      model.RecalculateModelParameters(problemData, rows);
+      return model;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegression.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegression.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegression.cs	(revision 17041)
@@ -0,0 +1,163 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Linear regression data analysis algorithm.
+  /// </summary>
+  [Item("Linear Regression (LR)", "Linear regression data analysis algorithm (wrapper for ALGLIB).")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 100)]
+  [StorableType("CF99D45E-F341-445E-9B9E-0587A8D9CBA7")]
+  public sealed class LinearRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+    private const string SolutionResultName = "Linear regression solution";
+    private const string ConfidenceSolutionResultName = "Solution with prediction intervals";
+
+    [StorableConstructor]
+    private LinearRegression(StorableConstructorFlag _) : base(_) { }
+    private LinearRegression(LinearRegression original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public LinearRegression()
+      : base() {
+      Problem = new RegressionProblem();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new LinearRegression(this, cloner);
+    }
+
+    #region linear regression
+    protected override void Run(CancellationToken cancellationToken) {
+      double rmsError, cvRmsError;
+      // produce both solutions, to allow symbolic manipulation of LR solutions as well
+      // as the calculation of prediction intervals.
+      // There is no clean way to implement the new model class for LR as a symbolic model.
+      var solution = CreateSolution(Problem.ProblemData, out rmsError, out cvRmsError);
+#pragma warning disable 168, 3021
+      var symbolicSolution = CreateLinearRegressionSolution(Problem.ProblemData, out rmsError, out cvRmsError);
+#pragma warning restore 168, 3021
+      Results.Add(new Result(SolutionResultName, "The linear regression solution.", symbolicSolution));
+      Results.Add(new Result(ConfidenceSolutionResultName, "Linear regression solution with parameter covariance matrix " +
+                                                           "and calculation of prediction intervals", solution));
+      Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the linear regression solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Estimated root mean square error (cross-validation)", "The estimated root of the mean of squared errors of the linear regression solution via cross validation.", new DoubleValue(cvRmsError)));
+    }
+
+    [Obsolete("Use CreateSolution() instead")]
+    public static ISymbolicRegressionSolution CreateLinearRegressionSolution(IRegressionProblemData problemData, out double rmsError, out double cvRmsError) {
+      IEnumerable<string> doubleVariables;
+      IEnumerable<KeyValuePair<string, IEnumerable<string>>> factorVariables;
+      double[,] inputMatrix;
+      PrepareData(problemData, out inputMatrix, out doubleVariables, out factorVariables);
+
+      alglib.linearmodel lm = new alglib.linearmodel();
+      alglib.lrreport ar = new alglib.lrreport();
+      int nRows = inputMatrix.GetLength(0);
+      int nFeatures = inputMatrix.GetLength(1) - 1;
+
+      int retVal = 1;
+      alglib.lrbuild(inputMatrix, nRows, nFeatures, out retVal, out lm, out ar);
+      if (retVal != 1) throw new ArgumentException("Error in calculation of linear regression solution");
+      rmsError = ar.rmserror;
+      cvRmsError = ar.cvrmserror;
+
+      double[] coefficients = new double[nFeatures + 1]; // last coefficient is for the constant
+      alglib.lrunpack(lm, out coefficients, out nFeatures);
+
+      int nFactorCoeff = factorVariables.Sum(kvp => kvp.Value.Count());
+      int nVarCoeff = doubleVariables.Count();
+      var tree = LinearModelToTreeConverter.CreateTree(factorVariables, coefficients.Take(nFactorCoeff).ToArray(),
+        doubleVariables.ToArray(), coefficients.Skip(nFactorCoeff).Take(nVarCoeff).ToArray(),
+        @const: coefficients[nFeatures]);
+
+      SymbolicRegressionSolution solution = new SymbolicRegressionSolution(new SymbolicRegressionModel(problemData.TargetVariable, tree, new SymbolicDataAnalysisExpressionTreeLinearInterpreter()), (IRegressionProblemData)problemData.Clone());
+      solution.Model.Name = "Linear Regression Model";
+      solution.Name = "Linear Regression Solution";
+      return solution;
+    }
+
+    public static IRegressionSolution CreateSolution(IRegressionProblemData problemData, out double rmsError, out double cvRmsError) {
+      IEnumerable<string> doubleVariables;
+      IEnumerable<KeyValuePair<string, IEnumerable<string>>> factorVariables;
+      double[,] inputMatrix;
+      PrepareData(problemData, out inputMatrix, out doubleVariables, out factorVariables);
+
+      alglib.linearmodel lm = new alglib.linearmodel();
+      alglib.lrreport ar = new alglib.lrreport();
+      int nRows = inputMatrix.GetLength(0);
+      int nFeatures = inputMatrix.GetLength(1) - 1;
+
+      int retVal = 1;
+      alglib.lrbuild(inputMatrix, nRows, nFeatures, out retVal, out lm, out ar);
+      if (retVal != 1) throw new ArgumentException("Error in calculation of linear regression solution");
+      rmsError = ar.rmserror;
+      cvRmsError = ar.cvrmserror;
+
+      // get parameters of the model
+      double[] w;
+      int nVars;
+      alglib.lrunpack(lm, out w, out nVars);
+
+      // ar.c is the covariation matrix,  array[0..NVars,0..NVars].
+      // C[i, j] = Cov(A[i], A[j])
+
+      var solution = new LinearRegressionModel(w, ar.c, cvRmsError, problemData.TargetVariable, doubleVariables, factorVariables)
+        .CreateRegressionSolution((IRegressionProblemData)problemData.Clone());
+      solution.Name = "Linear Regression Solution";
+      return solution;
+    }
+
+    private static void PrepareData(IRegressionProblemData problemData,
+      out double[,] inputMatrix,
+      out IEnumerable<string> doubleVariables,
+      out IEnumerable<KeyValuePair<string, IEnumerable<string>>> factorVariables) {
+      var dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+      doubleVariables = allowedInputVariables.Where(dataset.VariableHasType<double>);
+      var factorVariableNames = allowedInputVariables.Where(dataset.VariableHasType<string>);
+      factorVariables = dataset.GetFactorVariableValues(factorVariableNames, rows);
+      double[,] binaryMatrix = dataset.ToArray(factorVariables, rows);
+      double[,] doubleVarMatrix = dataset.ToArray(doubleVariables.Concat(new string[] { targetVariable }), rows);
+      inputMatrix = binaryMatrix.HorzCat(doubleVarMatrix);
+
+      if (inputMatrix.ContainsNanOrInfinity())
+        throw new NotSupportedException("Linear regression does not support NaN or infinity values in the input dataset.");
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegressionModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegressionModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegressionModel.cs	(revision 17041)
@@ -0,0 +1,160 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a linear regression model
+  /// </summary>
+  [StorableType("B65FB0CA-7333-41FE-8156-FF141C54F5AF")]
+  [Item("Linear Regression Model", "Represents a linear regression model.")]
+  public sealed class LinearRegressionModel : RegressionModel, IConfidenceRegressionModel {
+    public static new Image StaticItemImage {
+      get { return HeuristicLab.Common.Resources.VSImageLibrary.Function; }
+    }
+
+    [Storable]
+    public double[,] C {
+      get; private set;
+    }
+    [Storable]
+    public double[] W {
+      get; private set;
+    }
+
+    [Storable]
+    public double NoiseSigma {
+      get; private set;
+    }
+
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return doubleVariables.Union(factorVariables.Select(f => f.Key)); }
+    }
+
+    [Storable]
+    private string[] doubleVariables;
+    [Storable]
+    private List<KeyValuePair<string, IEnumerable<string>>> factorVariables;
+
+    /// <summary>
+    /// Enumerable of variable names used by the model including one-hot-encoded of factor variables.
+    /// </summary>
+    public IEnumerable<string> ParameterNames {
+      get {
+        return factorVariables.SelectMany(kvp => kvp.Value.Select(factorVal => $"{kvp.Key}={factorVal}"))
+          .Concat(doubleVariables)
+          .Concat(new[] { "<const>" });
+      }
+    }
+
+    [StorableConstructor]
+    private LinearRegressionModel(StorableConstructorFlag _) : base(_) {
+    }
+    private LinearRegressionModel(LinearRegressionModel original, Cloner cloner)
+      : base(original, cloner) {
+      this.W = original.W;
+      this.C = original.C;
+      this.NoiseSigma = original.NoiseSigma;
+
+      doubleVariables = (string[])original.doubleVariables.Clone();
+      this.factorVariables = original.factorVariables.Select(kvp => new KeyValuePair<string, IEnumerable<string>>(kvp.Key, new List<string>(kvp.Value))).ToList();
+    }
+    public LinearRegressionModel(double[] w, double[,] covariance, double noiseSigma, string targetVariable, IEnumerable<string> doubleInputVariables, IEnumerable<KeyValuePair<string, IEnumerable<string>>> factorVariables)
+      : base(targetVariable) {
+      this.name = ItemName;
+      this.description = ItemDescription;
+      this.W = new double[w.Length];
+      Array.Copy(w, W, w.Length);
+      this.C = new double[covariance.GetLength(0), covariance.GetLength(1)];
+      Array.Copy(covariance, C, covariance.Length);
+      this.NoiseSigma = noiseSigma;
+      this.doubleVariables = doubleInputVariables.ToArray();
+      // clone
+      this.factorVariables = factorVariables.Select(kvp => new KeyValuePair<string, IEnumerable<string>>(kvp.Key, new List<string>(kvp.Value))).ToList();
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new LinearRegressionModel(this, cloner);
+    }
+
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = dataset.ToArray(doubleVariables, rows);
+      double[,] factorData = dataset.ToArray(factorVariables, rows);
+
+      inputData = factorData.HorzCat(inputData);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+
+      for (int row = 0; row < n; row++) {
+        double p = 0.0;
+        for (int column = 0; column < columns; column++) {
+          p += W[column] * inputData[row, column];
+        }
+        p += W[columns];
+        yield return p;
+      }
+    }
+
+    public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = dataset.ToArray(doubleVariables, rows);
+      double[,] factorData = dataset.ToArray(factorVariables, rows);
+
+      inputData = factorData.HorzCat(inputData);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+
+      double[] d = new double[C.GetLength(0)];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          d[column] = inputData[row, column];
+        }
+        d[columns] = 1;
+
+        double var = 0.0;
+        for (int i = 0; i < d.Length; i++) {
+          for (int j = 0; j < d.Length; j++) {
+            var += d[i] * C[i, j] * d[j];
+          }
+        }
+        yield return var + NoiseSigma * NoiseSigma;
+      }
+    }
+
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new ConfidenceRegressionSolution(this, new RegressionProblemData(problemData));
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassification.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassification.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassification.cs	(revision 17041)
@@ -0,0 +1,109 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Multinomial logit regression data analysis algorithm.
+  /// </summary>
+  [Item("Multinomial Logit Classification (MNL)", "Multinomial logit classification data analysis algorithm (wrapper for ALGLIB).")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 180)]
+  [StorableType("F2797341-670A-491E-8652-0F154CBE99DC")]
+  public sealed class MultiNomialLogitClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+    private const string LogitClassificationModelResultName = "Logit classification solution";
+
+    [StorableConstructor]
+    private MultiNomialLogitClassification(StorableConstructorFlag _) : base(_) { }
+    private MultiNomialLogitClassification(MultiNomialLogitClassification original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public MultiNomialLogitClassification()
+      : base() {
+      Problem = new ClassificationProblem();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MultiNomialLogitClassification(this, cloner);
+    }
+
+    #region logit classification
+    protected override void Run(CancellationToken cancellationToken) {
+      double rmsError, relClassError;
+      var solution = CreateLogitClassificationSolution(Problem.ProblemData, out rmsError, out relClassError);
+      Results.Add(new Result(LogitClassificationModelResultName, "The logit classification solution.", solution));
+      Results.Add(new Result("Root mean squared error", "The root of the mean of squared errors of the logit regression solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Relative classification error", "Relative classification error on the training set (percentage of misclassified cases).", new PercentValue(relClassError)));
+    }
+
+    public static IClassificationSolution CreateLogitClassificationSolution(IClassificationProblemData problemData, out double rmsError, out double relClassError) {
+      var dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      var doubleVariableNames = problemData.AllowedInputVariables.Where(dataset.VariableHasType<double>);
+      var factorVariableNames = problemData.AllowedInputVariables.Where(dataset.VariableHasType<string>);
+      IEnumerable<int> rows = problemData.TrainingIndices;
+      double[,] inputMatrix = dataset.ToArray(doubleVariableNames.Concat(new string[] { targetVariable }), rows);
+
+      var factorVariableValues = dataset.GetFactorVariableValues(factorVariableNames, rows);
+      var factorMatrix = dataset.ToArray(factorVariableValues, rows);
+      inputMatrix = factorMatrix.HorzCat(inputMatrix);
+
+      if (inputMatrix.ContainsNanOrInfinity())
+        throw new NotSupportedException("Multinomial logit classification does not support NaN or infinity values in the input dataset.");
+
+      alglib.logitmodel lm = new alglib.logitmodel();
+      alglib.mnlreport rep = new alglib.mnlreport();
+      int nRows = inputMatrix.GetLength(0);
+      int nFeatures = inputMatrix.GetLength(1) - 1;
+      double[] classValues = dataset.GetDoubleValues(targetVariable).Distinct().OrderBy(x => x).ToArray();
+      int nClasses = classValues.Count();
+      // map original class values to values [0..nClasses-1]
+      Dictionary<double, double> classIndices = new Dictionary<double, double>();
+      for (int i = 0; i < nClasses; i++) {
+        classIndices[classValues[i]] = i;
+      }
+      for (int row = 0; row < nRows; row++) {
+        inputMatrix[row, nFeatures] = classIndices[inputMatrix[row, nFeatures]];
+      }
+      int info;
+      alglib.mnltrainh(inputMatrix, nRows, nFeatures, nClasses, out info, out lm, out rep);
+      if (info != 1) throw new ArgumentException("Error in calculation of logit classification solution");
+
+      rmsError = alglib.mnlrmserror(lm, inputMatrix, nRows);
+      relClassError = alglib.mnlrelclserror(lm, inputMatrix, nRows);
+
+      MultinomialLogitClassificationSolution solution = new MultinomialLogitClassificationSolution(new MultinomialLogitModel(lm, targetVariable, doubleVariableNames, factorVariableValues, classValues), (IClassificationProblemData)problemData.Clone());
+      return solution;
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassificationSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassificationSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassificationSolution.cs	(revision 17041)
@@ -0,0 +1,53 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a multinomial logit solution for a classification problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("Multinomial Logit Classification Solution", "Represents a multinomial logit solution for a classification problem which can be visualized in the GUI.")]
+  [StorableType("A6F6F990-EF97-4A9C-8054-B38E39763FDB")]
+  public sealed class MultinomialLogitClassificationSolution : ClassificationSolution {
+
+    public new MultinomialLogitModel Model {
+      get { return (MultinomialLogitModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private MultinomialLogitClassificationSolution(StorableConstructorFlag _) : base(_) { }
+    private MultinomialLogitClassificationSolution(MultinomialLogitClassificationSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public MultinomialLogitClassificationSolution(MultinomialLogitModel logitModel, IClassificationProblemData problemData)
+      : base(logitModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MultinomialLogitClassificationSolution(this, cloner);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitModel.cs	(revision 17041)
@@ -0,0 +1,151 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a multinomial logit model for classification
+  /// </summary>
+  [StorableType("AC4174A4-9FBC-4B07-9239-1E0E6F86034D")]
+  [Item("Multinomial Logit Model", "Represents a multinomial logit model for classification.")]
+  public sealed class MultinomialLogitModel : ClassificationModel {
+
+    private alglib.logitmodel logitModel;
+    public alglib.logitmodel Model {
+      get { return logitModel; }
+      set {
+        if (value != logitModel) {
+          if (value == null) throw new ArgumentNullException();
+          logitModel = value;
+          OnChanged(EventArgs.Empty);
+        }
+      }
+    }
+
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return allowedInputVariables; }
+    }
+
+    [Storable]
+    private string[] allowedInputVariables;
+    [Storable]
+    private double[] classValues;
+    [Storable]
+    private List<KeyValuePair<string, IEnumerable<string>>> factorVariables;
+
+    [StorableConstructor]
+    private MultinomialLogitModel(StorableConstructorFlag _) : base(_) {
+      logitModel = new alglib.logitmodel();
+    }
+    private MultinomialLogitModel(MultinomialLogitModel original, Cloner cloner)
+      : base(original, cloner) {
+      logitModel = new alglib.logitmodel();
+      logitModel.innerobj.w = (double[])original.logitModel.innerobj.w.Clone();
+      allowedInputVariables = (string[])original.allowedInputVariables.Clone();
+      classValues = (double[])original.classValues.Clone();
+      this.factorVariables = original.factorVariables.Select(kvp => new KeyValuePair<string, IEnumerable<string>>(kvp.Key, new List<string>(kvp.Value))).ToList();
+    }
+    public MultinomialLogitModel(alglib.logitmodel logitModel, string targetVariable, IEnumerable<string> doubleInputVariables, IEnumerable<KeyValuePair<string, IEnumerable<string>>> factorVariables, double[] classValues)
+      : base(targetVariable) {
+      this.name = ItemName;
+      this.description = ItemDescription;
+      this.logitModel = logitModel;
+      this.allowedInputVariables = doubleInputVariables.ToArray();
+      this.factorVariables = factorVariables.Select(kvp => new KeyValuePair<string, IEnumerable<string>>(kvp.Key, new List<string>(kvp.Value))).ToList();
+      this.classValues = (double[])classValues.Clone();
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
+      factorVariables = new List<KeyValuePair<string, IEnumerable<string>>>();
+      #endregion
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MultinomialLogitModel(this, cloner);
+    }
+
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+
+      double[,] inputData = dataset.ToArray(allowedInputVariables, rows);
+      double[,] factorData = dataset.ToArray(factorVariables, rows);
+
+      inputData = factorData.HorzCat(inputData);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] y = new double[classValues.Length];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        alglib.mnlprocess(logitModel, x, ref y);
+        // find class for with the largest probability value
+        int maxProbClassIndex = 0;
+        double maxProb = y[0];
+        for (int i = 1; i < y.Length; i++) {
+          if (maxProb < y[i]) {
+            maxProb = y[i];
+            maxProbClassIndex = i;
+          }
+        }
+        yield return classValues[maxProbClassIndex];
+      }
+    }
+
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new MultinomialLogitClassificationSolution(this, new ClassificationProblemData(problemData));
+    }
+
+    #region events
+    public event EventHandler Changed;
+    private void OnChanged(EventArgs e) {
+      var handlers = Changed;
+      if (handlers != null)
+        handlers(this, e);
+    }
+    #endregion
+
+    #region persistence
+    [Storable]
+    private double[] LogitModelW {
+      get {
+        return logitModel.innerobj.w;
+      }
+      set {
+        logitModel.innerobj.w = value;
+      }
+    }
+    #endregion
+
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/Scaling.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/Scaling.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/Scaling.cs	(revision 17041)
@@ -0,0 +1,69 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Obsolete("Use transformation classes in Problems.DataAnalysis instead")]
+  [StorableType("52EABC0F-B8D2-4ADD-ACC2-C825D3F1D6F3")]
+  [Item(Name = "Scaling", Description = "Contains information about scaling of variables for data-analysis algorithms.")]
+  public class Scaling : Item {
+    [Storable]
+    private Dictionary<string, Tuple<double, double>> scalingParameters = new Dictionary<string, Tuple<double, double>>();
+    [StorableConstructor]
+    protected Scaling(StorableConstructorFlag _) : base(_) { }
+    protected Scaling(Scaling original, Cloner cloner)
+      : base(original, cloner) {
+      foreach (var pair in original.scalingParameters)
+        scalingParameters.Add(pair.Key, Tuple.Create(pair.Value.Item1, pair.Value.Item2));
+    }
+    public Scaling(IDataset ds, IEnumerable<string> variables, IEnumerable<int> rows) {
+      foreach (var variable in variables) {
+        var values = ds.GetDoubleValues(variable, rows);
+        var min = values.Where(x => !double.IsNaN(x)).Min();
+        var max = values.Where(x => !double.IsNaN(x)).Max();
+        scalingParameters[variable] = Tuple.Create(min, max);
+      }
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new Scaling(this, cloner);
+    }
+
+    public IEnumerable<double> GetScaledValues(IDataset ds, string variable, IEnumerable<int> rows) {
+      double min = scalingParameters[variable].Item1;
+      double max = scalingParameters[variable].Item2;
+      if (min.IsAlmost(max)) return rows.Select(i => 0.0); // return enumerable of zeros
+      return ds.GetDoubleValues(variable, rows).Select(x => (x - min) / (max - min));  // scale to range [0..1]
+    }
+
+    public void GetScalingParameters(string variable, out double min, out double max) {
+      min = scalingParameters[variable].Item1;
+      max = scalingParameters[variable].Item2;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/INcaInitializer.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/INcaInitializer.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/INcaInitializer.cs	(revision 17041)
@@ -0,0 +1,37 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("4e8ff5de-c353-47cb-baa7-1a9be8f89456")]
+  public interface INcaInitializer : IOperator {
+    /// <summary>
+    /// Calculates an initial projection for the NCA to start from.
+    /// </summary>
+    /// <param name="data">The problem data that contains the AllowedInputVariables and TrainingIndices.</param>
+    /// <param name="dimensions">The amount of columns in the matrix</param>
+    /// <returns>The matrix that projects the input variables into a lower dimensional space.</returns>
+    double[,] Initialize(IClassificationProblemData data, int dimensions);
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/LdaInitializer.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/LdaInitializer.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/LdaInitializer.cs	(revision 17041)
@@ -0,0 +1,67 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("LDA", "Initializes the matrix by performing a linear discriminant analysis.")]
+  [StorableType("587DE65A-4BAD-4AC7-8C99-A9DE2B2A7B19")]
+  public class LdaInitializer : NcaInitializer {
+
+    [StorableConstructor]
+    protected LdaInitializer(StorableConstructorFlag _) : base(_) { }
+    protected LdaInitializer(LdaInitializer original, Cloner cloner) : base(original, cloner) { }
+    public LdaInitializer() : base() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new LdaInitializer(this, cloner);
+    }
+
+    public override double[,] Initialize(IClassificationProblemData data, int dimensions) {
+      var instances = data.TrainingIndices.Count();
+      var attributes = data.AllowedInputVariables.Count();
+
+      var ldaDs = data.Dataset.ToArray(
+                                       data.AllowedInputVariables.Concat(data.TargetVariable.ToEnumerable()),
+                                       data.TrainingIndices);
+
+      // map class values to sequential natural numbers (required by alglib)
+      var uniqueClasses = data.Dataset.GetDoubleValues(data.TargetVariable, data.TrainingIndices)
+                                        .Distinct()
+                                        .Select((v, i) => new { v, i })
+                                        .ToDictionary(x => x.v, x => x.i);
+
+      for (int row = 0; row < instances; row++)
+        ldaDs[row, attributes] = uniqueClasses[ldaDs[row, attributes]];
+
+      int info;
+      double[,] matrix;
+      alglib.fisherldan(ldaDs, instances, attributes, uniqueClasses.Count, out info, out matrix);
+
+      return matrix;
+    }
+
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/NcaInitializer.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/NcaInitializer.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/NcaInitializer.cs	(revision 17041)
@@ -0,0 +1,73 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Operators;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("NcaInitializer", "Base class for initializers for NCA.")]
+  [StorableType("165FEA5C-173F-46E3-AA38-16E125367094")]
+  public abstract class NcaInitializer : SingleSuccessorOperator, INcaInitializer {
+
+    public ILookupParameter<IClassificationProblemData> ProblemDataParameter {
+      get { return (ILookupParameter<IClassificationProblemData>)Parameters["ProblemData"]; }
+    }
+    public ILookupParameter<IntValue> DimensionsParameter {
+      get { return (ILookupParameter<IntValue>)Parameters["Dimensions"]; }
+    }
+    public ILookupParameter<RealVector> NcaMatrixParameter {
+      get { return (ILookupParameter<RealVector>)Parameters["NcaMatrix"]; }
+    }
+
+    [StorableConstructor]
+    protected NcaInitializer(StorableConstructorFlag _) : base(_) { }
+    protected NcaInitializer(NcaInitializer original, Cloner cloner) : base(original, cloner) { }
+    public NcaInitializer() {
+      Parameters.Add(new LookupParameter<IClassificationProblemData>("ProblemData", "The classification problem data."));
+      Parameters.Add(new LookupParameter<IntValue>("Dimensions", "The number of dimensions to which the features should be pruned."));
+      Parameters.Add(new LookupParameter<RealVector>("NcaMatrix", "The coefficients of the matrix that need to be optimized. Note that the matrix is flattened."));
+    }
+
+    public override IOperation Apply() {
+      var problemData = ProblemDataParameter.ActualValue;
+
+      var dimensions = DimensionsParameter.ActualValue.Value;
+      var matrix = Initialize(problemData, dimensions);
+      var attributes = matrix.GetLength(0);
+
+      var result = new double[attributes * dimensions];
+      for (int i = 0; i < attributes; i++)
+        for (int j = 0; j < dimensions; j++)
+          result[i * dimensions + j] = matrix[i, j];
+
+      NcaMatrixParameter.ActualValue = new RealVector(result);
+      return base.Apply();
+    }
+
+    public abstract double[,] Initialize(IClassificationProblemData data, int dimensions);
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/PcaInitializer.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/PcaInitializer.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/PcaInitializer.cs	(revision 17041)
@@ -0,0 +1,57 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("PCA", "Initializes the matrix by performing a principal components analysis.")]
+  [StorableType("C08B748E-61BC-4297-9ED0-383A41A699B4")]
+  public sealed class PcaInitializer : NcaInitializer {
+
+    [StorableConstructor]
+    private PcaInitializer(StorableConstructorFlag _) : base(_) { }
+    private PcaInitializer(PcaInitializer original, Cloner cloner) : base(original, cloner) { }
+    public PcaInitializer() : base() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new PcaInitializer(this, cloner);
+    }
+
+    public override double[,] Initialize(IClassificationProblemData data, int dimensions) {
+      var instances = data.TrainingIndices.Count();
+      var attributes = data.AllowedInputVariables.Count();
+
+      var pcaDs = data.Dataset.ToArray(data.AllowedInputVariables, data.TrainingIndices);
+
+      int info;
+      double[] varianceValues;
+      double[,] matrix;
+      alglib.pcabuildbasis(pcaDs, instances, attributes, out info, out varianceValues, out matrix);
+
+      return matrix;
+    }
+
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/RandomInitializer.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/RandomInitializer.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/RandomInitializer.cs	(revision 17041)
@@ -0,0 +1,62 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("Random", "Initializes the matrix randomly.")]
+  [StorableType("C799E0AA-1DC5-44BF-98F9-6CFEF5521BBC")]
+  public sealed class RandomInitializer : NcaInitializer, IStochasticOperator {
+    public ILookupParameter<IRandom> RandomParameter {
+      get { return (ILookupParameter<IRandom>)Parameters["Random"]; }
+    }
+
+    [StorableConstructor]
+    private RandomInitializer(StorableConstructorFlag _) : base(_) { }
+    private RandomInitializer(RandomInitializer original, Cloner cloner) : base(original, cloner) { }
+    public RandomInitializer()
+      : base() {
+      Parameters.Add(new LookupParameter<IRandom>("Random", "The random number generator to use."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RandomInitializer(this, cloner);
+    }
+
+    public override double[,] Initialize(IClassificationProblemData data, int dimensions) {
+      var attributes = data.AllowedInputVariables.Count();
+
+      var random = RandomParameter.ActualValue;
+      var matrix = new double[attributes, dimensions];
+      for (int i = 0; i < attributes; i++)
+        for (int j = 0; j < dimensions; j++)
+          matrix[i, j] = random.NextDouble();
+
+      return matrix;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Matrix.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Matrix.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Matrix.cs	(revision 17041)
@@ -0,0 +1,228 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.PluginInfrastructure;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [NonDiscoverableType]
+  internal class Matrix : IEnumerable<double>, IDeepCloneable {
+    // this type is immutable
+    private readonly IEnumerable<double> values;
+    public readonly int Rows;
+    public readonly int Columns;
+
+    protected Matrix(Matrix original, Cloner cloner) {
+      this.values = original.values.ToArray();
+      this.Rows = original.Rows;
+      this.Columns = original.Columns;
+      cloner.RegisterClonedObject(original, this);
+    }
+    public Matrix(IEnumerable<double> vector) {
+      this.values = vector;
+      Rows = 1;
+      Columns = vector.Count();
+    }
+    public Matrix(IEnumerable<double> vector, int length) {
+      this.values = vector;
+      Rows = 1;
+      Columns = length;
+    }
+    public Matrix(double[,] matrix) {
+      this.values = GetOnlineValues(matrix);
+      Rows = matrix.GetLength(0);
+      Columns = matrix.GetLength(1);
+    }
+    public Matrix(IEnumerable<double> matrix, int rows, int columns) {
+      this.values = matrix;
+      Rows = rows;
+      Columns = columns;
+    }
+
+    public object Clone() {
+      return Clone(new Cloner());
+    }
+    public IDeepCloneable Clone(Cloner cloner) {
+      return new Matrix(this, cloner);
+    }
+
+    public Matrix Transpose() {
+      return new Matrix(Transpose(values, Columns, Rows), Columns, Rows);
+    }
+
+    private IEnumerable<double> Transpose(IEnumerable<double> values, int rows, int columns) {
+      // vectors don't need to be transposed
+      if (rows == 1 || columns == 1) {
+        foreach (var v in values) yield return v;
+        yield break;
+      }
+
+      int skip = 0;
+      var iter = values.GetEnumerator();
+      if (!iter.MoveNext()) yield break;
+      while (skip < rows) {
+        for (int i = 0; i < skip; i++) iter.MoveNext();
+        yield return iter.Current;
+        for (int j = 0; j < columns - 1; j++) {
+          for (int i = 0; i < rows; i++) iter.MoveNext();
+          yield return iter.Current;
+        }
+        skip++;
+        if (skip < rows) {
+          iter = values.GetEnumerator();
+          iter.MoveNext();
+        }
+      }
+    }
+
+    public Matrix Add(Matrix other) {
+      return new Matrix(AddOnline(other), Rows, Columns);
+    }
+
+    public void AddTo(double[,] matrix) {
+      if (Rows != matrix.GetLength(0) || Columns != matrix.GetLength(1)) throw new ArgumentException("unequal size", "matrix");
+      var iter = values.GetEnumerator();
+      for (int i = 0; i < Rows; i++)
+        for (int j = 0; j < Columns; j++) {
+          iter.MoveNext();
+          matrix[i, j] += iter.Current;
+        }
+    }
+
+    public Matrix Subtract(Matrix other) {
+      return new Matrix(SubtractOnline(other), Rows, Columns);
+    }
+
+    public Matrix Multiply(Matrix other) {
+      return new Matrix(MultiplyOnline(other), Rows, other.Columns);
+    }
+
+    public Matrix Multiply(double value) {
+      return new Matrix(values.Select(x => x * value), Rows, Columns);
+    }
+
+    public double EuclideanNorm() {
+      return Math.Sqrt(SumOfSquares());
+    }
+
+    public double SumOfSquares() {
+      return values.Sum(x => x * x);
+    }
+
+    public Matrix OuterProduct(Matrix other) {
+      if (Rows != 1 || other.Rows != 1) throw new ArgumentException("OuterProduct can only be applied to vectors.");
+      return Transpose().Multiply(other);
+    }
+
+    public IEnumerable<double> ColumnSums() {
+      return Transpose().RowSums();
+    }
+
+    public IEnumerable<double> RowSums() {
+      var sum = 0.0;
+      int counter = 0;
+      foreach (var v in values) {
+        sum += v;
+        counter++;
+        if (counter == Rows) {
+          yield return sum;
+          sum = 0.0;
+          counter = 0;
+        }
+      }
+    }
+
+    public Matrix Negate() {
+      return new Matrix(values.Select(x => -x), Rows, Columns);
+    }
+
+    public Matrix Apply() {
+      return new Matrix(values.ToArray(), Rows, Columns);
+    }
+
+    public IEnumerator<double> GetEnumerator() { return values.GetEnumerator(); }
+    IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); }
+
+
+    private IEnumerable<double> AddOnline(Matrix other) {
+      if (Rows != other.Rows || Columns != other.Columns) throw new ArgumentException("Number of rows and columns are not equal.");
+      var meIter = values.GetEnumerator();
+      var otherIter = other.GetEnumerator();
+      if (!meIter.MoveNext()) yield break;
+      if (!otherIter.MoveNext()) yield break;
+      for (int i = 0; i < Rows * Columns; i++) {
+        yield return meIter.Current + otherIter.Current;
+        meIter.MoveNext();
+        otherIter.MoveNext();
+      }
+    }
+
+    private IEnumerable<double> SubtractOnline(Matrix other) {
+      if (Rows != other.Rows || Columns != other.Columns) throw new ArgumentException("Number of rows and columns are not equal.");
+      var meIter = values.GetEnumerator();
+      var otherIter = other.GetEnumerator();
+      if (!meIter.MoveNext()) yield break;
+      if (!otherIter.MoveNext()) yield break;
+      for (int i = 0; i < Rows * Columns; i++) {
+        yield return meIter.Current - otherIter.Current;
+        meIter.MoveNext();
+        otherIter.MoveNext();
+      }
+    }
+
+    private IEnumerable<double> MultiplyOnline(Matrix other) {
+      if (Columns != other.Rows) throw new ArgumentException("Number of rows and columns are not equal.");
+      var meIter = values.GetEnumerator();
+      var otherByColumn = other.Transpose();
+      var otherIter = otherByColumn.GetEnumerator();
+      if (!meIter.MoveNext()) yield break;
+      if (!otherIter.MoveNext()) yield break;
+      for (int r = 0; r < Rows; r++) {
+        var row = new double[Columns];
+        for (int x = 0; x < Columns; x++) {
+          row[x] = meIter.Current;
+          meIter.MoveNext();
+        }
+        for (int c = 0; c < other.Columns; c++) {
+          var sum = 0.0;
+          for (int y = 0; y < other.Rows; y++) {
+            sum += row[y] * otherIter.Current;
+            otherIter.MoveNext();
+          }
+          yield return sum;
+        }
+        otherIter = otherByColumn.GetEnumerator();
+        otherIter.MoveNext();
+      }
+    }
+
+    private IEnumerable<double> GetOnlineValues(double[,] matrix) {
+      for (int i = 0; i < matrix.GetLength(0); i++)
+        for (int j = 0; j < matrix.GetLength(1); j++) {
+          yield return matrix[i, j];
+        }
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/ModelCreation/INcaModelCreator.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/ModelCreation/INcaModelCreator.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/ModelCreation/INcaModelCreator.cs	(revision 17041)
@@ -0,0 +1,30 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("d93ecea5-379a-4569-8e0d-a80725d38d7f")]
+  public interface INcaModelCreator : IOperator {
+
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/ModelCreation/NcaModelCreator.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/ModelCreation/NcaModelCreator.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/ModelCreation/NcaModelCreator.cs	(revision 17041)
@@ -0,0 +1,93 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Operators;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("NcaModelCreator", "Creates an NCA model with a given matrix.")]
+  [StorableType("30D2840C-1FE3-4A45-97FF-294C93D33D8C")]
+  public class NcaModelCreator : SingleSuccessorOperator, INcaModelCreator {
+
+    public ILookupParameter<IntValue> KParameter {
+      get { return (ILookupParameter<IntValue>)Parameters["K"]; }
+    }
+
+    public ILookupParameter<IntValue> DimensionsParameter {
+      get { return (ILookupParameter<IntValue>)Parameters["Dimensions"]; }
+    }
+
+    public ILookupParameter<RealVector> NcaMatrixParameter {
+      get { return (ILookupParameter<RealVector>)Parameters["NcaMatrix"]; }
+    }
+
+    public ILookupParameter<RealVector> NcaMatrixGradientsParameter {
+      get { return (ILookupParameter<RealVector>)Parameters["NcaMatrixGradients"]; }
+    }
+
+    public ILookupParameter<IClassificationProblemData> ProblemDataParameter {
+      get { return (ILookupParameter<IClassificationProblemData>)Parameters["ProblemData"]; }
+    }
+
+    public ILookupParameter<INcaModel> NcaModelParameter {
+      get { return (ILookupParameter<INcaModel>)Parameters["NcaModel"]; }
+    }
+
+    [StorableConstructor]
+    protected NcaModelCreator(StorableConstructorFlag _) : base(_) { }
+    protected NcaModelCreator(NcaModelCreator original, Cloner cloner) : base(original, cloner) { }
+    public NcaModelCreator() {
+      Parameters.Add(new LookupParameter<IntValue>("K", "How many neighbors should be considered for classification."));
+      Parameters.Add(new LookupParameter<IntValue>("Dimensions", "The dimensions to which the feature space should be reduced to."));
+      Parameters.Add(new LookupParameter<RealVector>("NcaMatrix", "The optimized matrix."));
+      Parameters.Add(new LookupParameter<RealVector>("NcaMatrixGradients", "The gradients from the matrix that is being optimized."));
+      Parameters.Add(new LookupParameter<IClassificationProblemData>("ProblemData", "The classification problem data."));
+      Parameters.Add(new LookupParameter<INcaModel>("NcaModel", "The NCA model that should be created."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NcaModelCreator(this, cloner);
+    }
+
+    public override IOperation Apply() {
+      var k = KParameter.ActualValue.Value;
+      var dim = DimensionsParameter.ActualValue.Value;
+      var vector = NcaMatrixParameter.ActualValue;
+      var matrix = new double[vector.Length / dim, dim];
+
+      for (int i = 0; i < matrix.GetLength(0); i++)
+        for (int j = 0; j < dim; j++) {
+          matrix[i, j] = vector[i * dim + j];
+        }
+
+      var problemData = ProblemDataParameter.ActualValue;
+      NcaModelParameter.ActualValue = new NcaModel(k, matrix, problemData.Dataset, problemData.TrainingIndices, problemData.TargetVariable, problemData.AllowedInputVariables, problemData.ClassValues.ToArray());
+      return base.Apply();
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaAlgorithm.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaAlgorithm.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaAlgorithm.cs	(revision 17041)
@@ -0,0 +1,254 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using HeuristicLab.Algorithms.GradientDescent;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Operators;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.PluginInfrastructure;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Random;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Neighborhood Components Analysis
+  /// </summary>
+  [Item("Neighborhood Components Analysis (NCA)", @"Implementation of Neighborhood Components Analysis
+based on the description of J. Goldberger, S. Roweis, G. Hinton, R. Salakhutdinov. 2005.
+Neighbourhood Component Analysis. Advances in Neural Information Processing Systems, 17. pp. 513-520
+with additional regularizations described in Z. Yang, J. Laaksonen. 2007.
+Regularized Neighborhood Component Analysis. Lecture Notes in Computer Science, 4522. pp. 253-262.")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 170)]
+  [StorableType("3C489916-B35E-408D-B225-CAA41B156065")]
+  public sealed class NcaAlgorithm : EngineAlgorithm {
+    #region Parameter Names
+    private const string SeedParameterName = "Seed";
+    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+    private const string KParameterName = "K";
+    private const string DimensionsParameterName = "Dimensions";
+    private const string InitializationParameterName = "Initialization";
+    private const string NeighborSamplesParameterName = "NeighborSamples";
+    private const string IterationsParameterName = "Iterations";
+    private const string RegularizationParameterName = "Regularization";
+    private const string NcaModelCreatorParameterName = "NcaModelCreator";
+    private const string NcaSolutionCreatorParameterName = "NcaSolutionCreator";
+    private const string ApproximateGradientsParameterName = "ApproximateGradients";
+    private const string NcaMatrixParameterName = "NcaMatrix";
+    private const string NcaMatrixGradientsParameterName = "NcaMatrixGradients";
+    private const string QualityParameterName = "Quality";
+    #endregion
+
+    public override Type ProblemType { get { return typeof(IClassificationProblem); } }
+    public new IClassificationProblem Problem {
+      get { return (IClassificationProblem)base.Problem; }
+      set { base.Problem = value; }
+    }
+
+    #region Parameter Properties
+    public IValueParameter<IntValue> SeedParameter {
+      get { return (IValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+    public IValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (IValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> KParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[KParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> DimensionsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[DimensionsParameterName]; }
+    }
+    public IConstrainedValueParameter<INcaInitializer> InitializationParameter {
+      get { return (IConstrainedValueParameter<INcaInitializer>)Parameters[InitializationParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NeighborSamplesParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NeighborSamplesParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> IterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[IterationsParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> RegularizationParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[RegularizationParameterName]; }
+    }
+    public IValueParameter<BoolValue> ApproximateGradientsParameter {
+      get { return (IValueParameter<BoolValue>)Parameters[ApproximateGradientsParameterName]; }
+    }
+    public IValueParameter<INcaModelCreator> NcaModelCreatorParameter {
+      get { return (IValueParameter<INcaModelCreator>)Parameters[NcaModelCreatorParameterName]; }
+    }
+    public IValueParameter<INcaSolutionCreator> NcaSolutionCreatorParameter {
+      get { return (IValueParameter<INcaSolutionCreator>)Parameters[NcaSolutionCreatorParameterName]; }
+    }
+    #endregion
+
+    #region Properties
+    public int Seed {
+      get { return SeedParameter.Value.Value; }
+      set { SeedParameter.Value.Value = value; }
+    }
+    public bool SetSeedRandomly {
+      get { return SetSeedRandomlyParameter.Value.Value; }
+      set { SetSeedRandomlyParameter.Value.Value = value; }
+    }
+    public int K {
+      get { return KParameter.Value.Value; }
+      set { KParameter.Value.Value = value; }
+    }
+    public int Dimensions {
+      get { return DimensionsParameter.Value.Value; }
+      set { DimensionsParameter.Value.Value = value; }
+    }
+    public int NeighborSamples {
+      get { return NeighborSamplesParameter.Value.Value; }
+      set { NeighborSamplesParameter.Value.Value = value; }
+    }
+    public int Iterations {
+      get { return IterationsParameter.Value.Value; }
+      set { IterationsParameter.Value.Value = value; }
+    }
+    public double Regularization {
+      get { return RegularizationParameter.Value.Value; }
+      set { RegularizationParameter.Value.Value = value; }
+    }
+    public INcaModelCreator NcaModelCreator {
+      get { return NcaModelCreatorParameter.Value; }
+      set { NcaModelCreatorParameter.Value = value; }
+    }
+    public INcaSolutionCreator NcaSolutionCreator {
+      get { return NcaSolutionCreatorParameter.Value; }
+      set { NcaSolutionCreatorParameter.Value = value; }
+    }
+    #endregion
+
+    [StorableConstructor]
+    private NcaAlgorithm(StorableConstructorFlag _) : base(_) { }
+    private NcaAlgorithm(NcaAlgorithm original, Cloner cloner) : base(original, cloner) { }
+    public NcaAlgorithm()
+      : base() {
+      Parameters.Add(new ValueParameter<IntValue>(SeedParameterName, "The seed of the random number generator.", new IntValue(0)));
+      Parameters.Add(new ValueParameter<BoolValue>(SetSeedRandomlyParameterName, "A boolean flag that indicates whether the seed should be randomly reset each time the algorithm is run.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<IntValue>(KParameterName, "The K for the nearest neighbor.", new IntValue(3)));
+      Parameters.Add(new FixedValueParameter<IntValue>(DimensionsParameterName, "The number of dimensions that NCA should reduce the data to.", new IntValue(2)));
+      Parameters.Add(new ConstrainedValueParameter<INcaInitializer>(InitializationParameterName, "Which method should be used to initialize the matrix. Typically LDA (linear discriminant analysis) should provide a good estimate."));
+      Parameters.Add(new FixedValueParameter<IntValue>(NeighborSamplesParameterName, "How many of the neighbors should be sampled in order to speed up the calculation. This should be at least the value of k and at most the number of training instances minus one will be used.", new IntValue(60)));
+      Parameters.Add(new FixedValueParameter<IntValue>(IterationsParameterName, "How many iterations the conjugate gradient (CG) method should be allowed to perform. The method might still terminate earlier if a local optima has already been reached.", new IntValue(50)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(RegularizationParameterName, "A non-negative paramter which can be set to increase generalization and avoid overfitting. If set to 0 the algorithm is similar to NCA as proposed by Goldberger et al.", new DoubleValue(0)));
+      Parameters.Add(new ValueParameter<INcaModelCreator>(NcaModelCreatorParameterName, "Creates an NCA model out of the matrix.", new NcaModelCreator()));
+      Parameters.Add(new ValueParameter<INcaSolutionCreator>(NcaSolutionCreatorParameterName, "Creates an NCA solution given a model and some data.", new NcaSolutionCreator()));
+      Parameters.Add(new ValueParameter<BoolValue>(ApproximateGradientsParameterName, "True if the gradient should be approximated otherwise they are computed exactly.", new BoolValue()));
+
+      NcaSolutionCreatorParameter.Hidden = true;
+      ApproximateGradientsParameter.Hidden = true;
+
+      INcaInitializer defaultInitializer = null;
+      foreach (var initializer in ApplicationManager.Manager.GetInstances<INcaInitializer>().OrderBy(x => x.ItemName)) {
+        if (initializer is LdaInitializer) defaultInitializer = initializer;
+        InitializationParameter.ValidValues.Add(initializer);
+      }
+      if (defaultInitializer != null) InitializationParameter.Value = defaultInitializer;
+
+      var randomCreator = new RandomCreator();
+      var ncaInitializer = new Placeholder();
+      var bfgsInitializer = new LbfgsInitializer();
+      var makeStep = new LbfgsMakeStep();
+      var branch = new ConditionalBranch();
+      var gradientCalculator = new NcaGradientCalculator();
+      var modelCreator = new Placeholder();
+      var updateResults = new LbfgsUpdateResults();
+      var analyzer = new LbfgsAnalyzer();
+      var finalModelCreator = new Placeholder();
+      var finalAnalyzer = new LbfgsAnalyzer();
+      var solutionCreator = new Placeholder();
+
+      OperatorGraph.InitialOperator = randomCreator;
+      randomCreator.SeedParameter.ActualName = SeedParameterName;
+      randomCreator.SeedParameter.Value = null;
+      randomCreator.SetSeedRandomlyParameter.ActualName = SetSeedRandomlyParameterName;
+      randomCreator.SetSeedRandomlyParameter.Value = null;
+      randomCreator.Successor = ncaInitializer;
+
+      ncaInitializer.Name = "(NcaInitializer)";
+      ncaInitializer.OperatorParameter.ActualName = InitializationParameterName;
+      ncaInitializer.Successor = bfgsInitializer;
+
+      bfgsInitializer.IterationsParameter.ActualName = IterationsParameterName;
+      bfgsInitializer.PointParameter.ActualName = NcaMatrixParameterName;
+      bfgsInitializer.ApproximateGradientsParameter.ActualName = ApproximateGradientsParameterName;
+      bfgsInitializer.Successor = makeStep;
+
+      makeStep.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
+      makeStep.PointParameter.ActualName = NcaMatrixParameterName;
+      makeStep.Successor = branch;
+
+      branch.ConditionParameter.ActualName = makeStep.TerminationCriterionParameter.Name;
+      branch.FalseBranch = gradientCalculator;
+      branch.TrueBranch = finalModelCreator;
+
+      gradientCalculator.Successor = modelCreator;
+
+      modelCreator.OperatorParameter.ActualName = NcaModelCreatorParameterName;
+      modelCreator.Successor = updateResults;
+
+      updateResults.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
+      updateResults.QualityParameter.ActualName = QualityParameterName;
+      updateResults.QualityGradientsParameter.ActualName = NcaMatrixGradientsParameterName;
+      updateResults.ApproximateGradientsParameter.ActualName = ApproximateGradientsParameterName;
+      updateResults.Successor = analyzer;
+
+      analyzer.QualityParameter.ActualName = QualityParameterName;
+      analyzer.PointParameter.ActualName = NcaMatrixParameterName;
+      analyzer.QualityGradientsParameter.ActualName = NcaMatrixGradientsParameterName;
+      analyzer.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
+      analyzer.PointsTableParameter.ActualName = "Matrix table";
+      analyzer.QualityGradientsTableParameter.ActualName = "Gradients table";
+      analyzer.QualitiesTableParameter.ActualName = "Qualities";
+      analyzer.Successor = makeStep;
+
+      finalModelCreator.OperatorParameter.ActualName = NcaModelCreatorParameterName;
+      finalModelCreator.Successor = finalAnalyzer;
+
+      finalAnalyzer.QualityParameter.ActualName = QualityParameterName;
+      finalAnalyzer.PointParameter.ActualName = NcaMatrixParameterName;
+      finalAnalyzer.QualityGradientsParameter.ActualName = NcaMatrixGradientsParameterName;
+      finalAnalyzer.PointsTableParameter.ActualName = analyzer.PointsTableParameter.ActualName;
+      finalAnalyzer.QualityGradientsTableParameter.ActualName = analyzer.QualityGradientsTableParameter.ActualName;
+      finalAnalyzer.QualitiesTableParameter.ActualName = analyzer.QualitiesTableParameter.ActualName;
+      finalAnalyzer.Successor = solutionCreator;
+
+      solutionCreator.OperatorParameter.ActualName = NcaSolutionCreatorParameterName;
+
+      Problem = new ClassificationProblem();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NcaAlgorithm(this, cloner);
+    }
+
+    public override void Prepare() {
+      if (Problem != null) base.Prepare();
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaClassificationSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaClassificationSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaClassificationSolution.cs	(revision 17041)
@@ -0,0 +1,50 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("NCA Classification Solution", "")]
+  [StorableType("3FE3A37F-2926-43EB-AC77-3C80654D93AA")]
+  public class NcaClassificationSolution : ClassificationSolution, INcaClassificationSolution {
+
+    public new INcaModel Model {
+      get { return (INcaModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    protected NcaClassificationSolution(StorableConstructorFlag _) : base(_) { }
+    protected NcaClassificationSolution(NcaClassificationSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NcaClassificationSolution(INcaModel ncaModel, IClassificationProblemData problemData)
+      : base(ncaModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NcaClassificationSolution(this, cloner);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaGradientCalculator.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaGradientCalculator.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaGradientCalculator.cs	(revision 17041)
@@ -0,0 +1,174 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Operators;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("NcaGradientCalculator", "Calculates the quality and gradient of a certain NCA matrix.")]
+  [StorableType("51A6EEB2-321D-460A-AF45-414144E06C85")]
+  public class NcaGradientCalculator : SingleSuccessorOperator, ISingleObjectiveOperator {
+
+    #region Parameter Properties
+    public ILookupParameter<IntValue> DimensionsParameter {
+      get { return (ILookupParameter<IntValue>)Parameters["Dimensions"]; }
+    }
+
+    public ILookupParameter<IntValue> NeighborSamplesParameter {
+      get { return (ILookupParameter<IntValue>)Parameters["NeighborSamples"]; }
+    }
+
+    public ILookupParameter<DoubleValue> RegularizationParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters["Regularization"]; }
+    }
+
+    public ILookupParameter<RealVector> NcaMatrixParameter {
+      get { return (ILookupParameter<RealVector>)Parameters["NcaMatrix"]; }
+    }
+
+    public ILookupParameter<RealVector> NcaMatrixGradientsParameter {
+      get { return (ILookupParameter<RealVector>)Parameters["NcaMatrixGradients"]; }
+    }
+
+    public ILookupParameter<DoubleValue> QualityParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters["Quality"]; }
+    }
+
+    public ILookupParameter<IClassificationProblemData> ProblemDataParameter {
+      get { return (ILookupParameter<IClassificationProblemData>)Parameters["ProblemData"]; }
+    }
+    #endregion
+
+    [StorableConstructor]
+    protected NcaGradientCalculator(StorableConstructorFlag _) : base(_) { }
+    protected NcaGradientCalculator(NcaGradientCalculator original, Cloner cloner) : base(original, cloner) { }
+    public NcaGradientCalculator()
+      : base() {
+      Parameters.Add(new LookupParameter<IntValue>("Dimensions", "The dimensions to which the feature space should be reduced to."));
+      Parameters.Add(new LookupParameter<IntValue>("NeighborSamples", "The number of neighbors that should be taken into account at maximum."));
+      Parameters.Add(new LookupParameter<DoubleValue>("Regularization", "The regularization term that constrains the expansion of the projected space."));
+      Parameters.Add(new LookupParameter<RealVector>("NcaMatrix", "The optimized matrix."));
+      Parameters.Add(new LookupParameter<RealVector>("NcaMatrixGradients", "The gradients from the matrix that is being optimized."));
+      Parameters.Add(new LookupParameter<DoubleValue>("Quality", "The quality of the current matrix."));
+      Parameters.Add(new LookupParameter<IClassificationProblemData>("ProblemData", "The classification problem data."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NcaGradientCalculator(this, cloner);
+    }
+
+    public override IOperation Apply() {
+      var problemData = ProblemDataParameter.ActualValue;
+      var dimensions = DimensionsParameter.ActualValue.Value;
+      var neighborSamples = NeighborSamplesParameter.ActualValue.Value;
+      var regularization = RegularizationParameter.ActualValue.Value;
+
+      var vector = NcaMatrixParameter.ActualValue;
+      var gradients = NcaMatrixGradientsParameter.ActualValue;
+      if (gradients == null) {
+        gradients = new RealVector(vector.Length);
+        NcaMatrixGradientsParameter.ActualValue = gradients;
+      }
+
+      var data = problemData.Dataset.ToArray(problemData.AllowedInputVariables,
+                                             problemData.TrainingIndices);
+      var classes = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices).ToArray();
+
+      var quality = Gradient(vector, gradients, data, classes, dimensions, neighborSamples, regularization);
+      QualityParameter.ActualValue = new DoubleValue(quality);
+
+      return base.Apply();
+    }
+
+    private static double Gradient(RealVector A, RealVector grad, double[,] data, double[] classes, int dimensions, int neighborSamples, double regularization) {
+      var instances = data.GetLength(0);
+      var attributes = data.GetLength(1);
+
+      var AMatrix = new Matrix(A, A.Length / dimensions, dimensions);
+
+      alglib.sparsematrix probabilities;
+      alglib.sparsecreate(instances, instances, out probabilities);
+      var transformedDistances = new Dictionary<int, double>(instances);
+      for (int i = 0; i < instances; i++) {
+        var iVector = new Matrix(GetRow(data, i), data.GetLength(1));
+        for (int k = 0; k < instances; k++) {
+          if (k == i) {
+            transformedDistances.Remove(k);
+            continue;
+          }
+          var kVector = new Matrix(GetRow(data, k));
+          transformedDistances[k] = Math.Exp(-iVector.Multiply(AMatrix).Subtract(kVector.Multiply(AMatrix)).SumOfSquares());
+        }
+        var normalization = transformedDistances.Sum(x => x.Value);
+        if (normalization <= 0) continue;
+        foreach (var s in transformedDistances.Where(x => x.Value > 0).OrderByDescending(x => x.Value).Take(neighborSamples)) {
+          alglib.sparseset(probabilities, i, s.Key, s.Value / normalization);
+        }
+      }
+      alglib.sparseconverttocrs(probabilities); // needed to enumerate in order (top-down and left-right)
+
+      int t0 = 0, t1 = 0, r, c;
+      double val;
+      var pi = new double[instances];
+      while (alglib.sparseenumerate(probabilities, ref t0, ref t1, out r, out c, out val)) {
+        if (classes[r].IsAlmost(classes[c])) {
+          pi[r] += val;
+        }
+      }
+
+      var innerSum = new double[attributes, attributes];
+      while (alglib.sparseenumerate(probabilities, ref t0, ref t1, out r, out c, out val)) {
+        var vector = new Matrix(GetRow(data, r)).Subtract(new Matrix(GetRow(data, c)));
+        vector.OuterProduct(vector).Multiply(val * pi[r]).AddTo(innerSum);
+
+        if (classes[r].IsAlmost(classes[c])) {
+          vector.OuterProduct(vector).Multiply(-val).AddTo(innerSum);
+        }
+      }
+
+      var func = -pi.Sum() + regularization * AMatrix.SumOfSquares();
+
+      r = 0;
+      var newGrad = AMatrix.Multiply(-2.0).Transpose().Multiply(new Matrix(innerSum)).Transpose();
+      foreach (var g in newGrad) {
+        grad[r] = g + regularization * 2 * A[r];
+        r++;
+      }
+
+      return func;
+    }
+
+    private static IEnumerable<double> GetRow(double[,] data, int row) {
+      for (int i = 0; i < data.GetLength(1); i++)
+        yield return data[row, i];
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs	(revision 17041)
@@ -0,0 +1,110 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("NCA Model", "")]
+  [StorableType("BB2F9ECA-CEAF-4983-A34C-9A42A132B7CD")]
+  public class NcaModel : ClassificationModel, INcaModel {
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return allowedInputVariables; }
+    }
+
+    [Storable]
+    private double[,] transformationMatrix;
+    public double[,] TransformationMatrix {
+      get { return (double[,])transformationMatrix.Clone(); }
+    }
+    [Storable]
+    private string[] allowedInputVariables;
+    [Storable]
+    private INearestNeighbourModel nnModel;
+    [Storable]
+    private double[] classValues;
+
+    [StorableConstructor]
+    protected NcaModel(StorableConstructorFlag _) : base(_) { }
+    protected NcaModel(NcaModel original, Cloner cloner)
+      : base(original, cloner) {
+      this.transformationMatrix = (double[,])original.transformationMatrix.Clone();
+      this.allowedInputVariables = (string[])original.allowedInputVariables.Clone();
+      this.nnModel = cloner.Clone(original.nnModel);
+      this.classValues = (double[])original.classValues.Clone();
+    }
+    public NcaModel(int k, double[,] transformationMatrix, IDataset dataset, IEnumerable<int> rows, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues)
+      : base(targetVariable) {
+      Name = ItemName;
+      Description = ItemDescription;
+      this.transformationMatrix = (double[,])transformationMatrix.Clone();
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+      this.classValues = (double[])classValues.Clone();
+
+      var ds = ReduceDataset(dataset, rows);
+      nnModel = new NearestNeighbourModel(ds, Enumerable.Range(0, ds.Rows), k, false, ds.VariableNames.Last(), ds.VariableNames.Take(transformationMatrix.GetLength(1)), classValues: classValues);
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NcaModel(this, cloner);
+    }
+
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+      var ds = ReduceDataset(dataset, rows);
+      return nnModel.GetEstimatedClassValues(ds, Enumerable.Range(0, ds.Rows));
+    }
+
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new NcaClassificationSolution(this, new ClassificationProblemData(problemData));
+    }
+
+    INcaClassificationSolution INcaModel.CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new NcaClassificationSolution(this, new ClassificationProblemData(problemData));
+    }
+
+    public double[,] Reduce(IDataset dataset, IEnumerable<int> rows) {
+      var data = dataset.ToArray(allowedInputVariables, rows);
+
+      var targets = dataset.GetDoubleValues(TargetVariable, rows).ToArray();
+      var result = new double[data.GetLength(0), transformationMatrix.GetLength(1) + 1];
+      for (int i = 0; i < data.GetLength(0); i++)
+        for (int j = 0; j < data.GetLength(1); j++) {
+          for (int x = 0; x < transformationMatrix.GetLength(1); x++) {
+            result[i, x] += data[i, j] * transformationMatrix[j, x];
+          }
+          result[i, transformationMatrix.GetLength(1)] = targets[i];
+        }
+      return result;
+    }
+
+    public Dataset ReduceDataset(IDataset dataset, IEnumerable<int> rows) {
+      return new Dataset(Enumerable
+          .Range(0, transformationMatrix.GetLength(1))
+          .Select(x => "X" + x.ToString())
+          .Concat(TargetVariable.ToEnumerable()),
+        Reduce(dataset, rows));
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/SolutionCreation/INcaSolutionCreator.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/SolutionCreation/INcaSolutionCreator.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/SolutionCreation/INcaSolutionCreator.cs	(revision 17041)
@@ -0,0 +1,30 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("0f9ff0e5-54b8-4bcd-8e1f-a3e92260803d")]
+  public interface INcaSolutionCreator : IOperator {
+
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/SolutionCreation/NcaSolutionCreator.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/SolutionCreation/NcaSolutionCreator.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/SolutionCreation/NcaSolutionCreator.cs	(revision 17041)
@@ -0,0 +1,91 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Operators;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("NcaSolutionCreator", "Creates an NCA solution with a given model and some given data.")]
+  [StorableType("BAAD907C-B2A2-40C8-BBEE-F8EFF12A795D")]
+  public class NcaSolutionCreator : SingleSuccessorOperator, INcaSolutionCreator {
+
+    public ILookupParameter<IClassificationProblemData> ProblemDataParameter {
+      get { return (ILookupParameter<IClassificationProblemData>)Parameters["ProblemData"]; }
+    }
+
+    public ILookupParameter<INcaModel> NcaModelParameter {
+      get { return (ILookupParameter<INcaModel>)Parameters["NcaModel"]; }
+    }
+
+    public ILookupParameter<INcaClassificationSolution> NcaSolutionParameter {
+      get { return (ILookupParameter<INcaClassificationSolution>)Parameters["NcaSolution"]; }
+    }
+
+    public ILookupParameter<ResultCollection> ResultsParameter {
+      get { return (ILookupParameter<ResultCollection>)Parameters["Results"]; }
+    }
+
+    [StorableConstructor]
+    protected NcaSolutionCreator(StorableConstructorFlag _) : base(_) { }
+    protected NcaSolutionCreator(NcaSolutionCreator original, Cloner cloner) : base(original, cloner) { }
+    public NcaSolutionCreator()
+      : base() {
+      Parameters.Add(new LookupParameter<IClassificationProblemData>("ProblemData", "The classification problem data."));
+      Parameters.Add(new LookupParameter<INcaModel>("NcaModel", "The NCA model that should be created."));
+      Parameters.Add(new LookupParameter<INcaClassificationSolution>("NcaSolution", "The created NCA solution."));
+      Parameters.Add(new LookupParameter<ResultCollection>("Results", "The results collection to store the results."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NcaSolutionCreator(this, cloner);
+    }
+
+    public override IOperation Apply() {
+      var problemData = ProblemDataParameter.ActualValue;
+      var model = NcaModelParameter.ActualValue;
+      var results = ResultsParameter.ActualValue;
+
+      var s = model.CreateClassificationSolution(problemData);
+      NcaSolutionParameter.ActualValue = s;
+
+      if (!results.ContainsKey("Solution")) {
+        results.Add(new Result("Solution", "The NCA classification solution", s));
+        results.Add(new Result("Accuracy (training)",
+                               "The accuracy of the NCA solution on the training partition.",
+                               new DoubleValue(s.TrainingAccuracy)));
+        results.Add(new Result("Accuracy (test)",
+                               "The accuracy of the NCA solution on the test partition.",
+                               new DoubleValue(s.TestAccuracy)));
+      } else {
+        results["Solution"].Value = s;
+        results["Accuracy (training)"].Value = new DoubleValue(s.TrainingAccuracy);
+        results["Accuracy (test)"].Value = new DoubleValue(s.TestAccuracy);
+      }
+      return base.Apply();
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs	(revision 17041)
@@ -0,0 +1,129 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Nearest neighbour classification data analysis algorithm.
+  /// </summary>
+  [Item("Nearest Neighbour Classification (kNN)", "Nearest neighbour classification data analysis algorithm (wrapper for ALGLIB).")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 150)]
+  [StorableType("98161D6F-D977-45EA-B899-E47EE017865E")]
+  public sealed class NearestNeighbourClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+    private const string KParameterName = "K";
+    private const string NearestNeighbourClassificationModelResultName = "Nearest neighbour classification solution";
+    private const string WeightsParameterName = "Weights";
+    private const string SelfMatchParameterName = "SelfMatch";
+
+    #region parameter properties
+    public IFixedValueParameter<IntValue> KParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[KParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> SelfMatchParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[SelfMatchParameterName]; }
+    }
+    public IValueParameter<DoubleArray> WeightsParameter {
+      get { return (IValueParameter<DoubleArray>)Parameters[WeightsParameterName]; }
+    }
+    #endregion
+    #region properties
+    public bool SelfMatch {
+      get { return SelfMatchParameter.Value.Value; }
+      set { SelfMatchParameter.Value.Value = value; }
+    }
+    public int K {
+      get { return KParameter.Value.Value; }
+      set {
+        if (value <= 0) throw new ArgumentException("K must be larger than zero.", "K");
+        else KParameter.Value.Value = value;
+      }
+    }
+    public DoubleArray Weights {
+      get { return WeightsParameter.Value; }
+      set { WeightsParameter.Value = value; }
+    }
+    #endregion
+
+    [StorableConstructor]
+    private NearestNeighbourClassification(StorableConstructorFlag _) : base(_) { }
+    private NearestNeighbourClassification(NearestNeighbourClassification original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NearestNeighbourClassification()
+      : base() {
+      Parameters.Add(new FixedValueParameter<BoolValue>(SelfMatchParameterName, "Should we use equal points for classification?", new BoolValue(false)));
+      Parameters.Add(new FixedValueParameter<IntValue>(KParameterName, "The number of nearest neighbours to consider for regression.", new IntValue(3)));
+      Parameters.Add(new OptionalValueParameter<DoubleArray>(WeightsParameterName, "Optional: use weights to specify individual scaling values for all features. If not set the weights are calculated automatically (each feature is scaled to unit variance)"));
+      Problem = new ClassificationProblem();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
+      if (!Parameters.ContainsKey(WeightsParameterName)) {
+        Parameters.Add(new OptionalValueParameter<DoubleArray>(WeightsParameterName, "Optional: use weights to specify individual scaling values for all features. If not set the weights are calculated automatically (each feature is scaled to unit variance)"));
+      }
+      if (!Parameters.ContainsKey(SelfMatchParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(SelfMatchParameterName, "Should we use equal points for classification?", new BoolValue(false)));
+      }
+      #endregion
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NearestNeighbourClassification(this, cloner);
+    }
+
+    #region nearest neighbour
+    protected override void Run(CancellationToken cancellationToken) {
+      double[] weights = null;
+      if (Weights != null) weights = Weights.CloneAsArray();
+      var solution = CreateNearestNeighbourClassificationSolution(Problem.ProblemData, K, SelfMatch, weights);
+      Results.Add(new Result(NearestNeighbourClassificationModelResultName, "The nearest neighbour classification solution.", solution));
+    }
+
+    public static IClassificationSolution CreateNearestNeighbourClassificationSolution(IClassificationProblemData problemData, int k, bool selfMatch = false, double[] weights = null) {
+      var problemDataClone = (IClassificationProblemData)problemData.Clone();
+      return new NearestNeighbourClassificationSolution(Train(problemDataClone, k, selfMatch, weights), problemDataClone);
+    }
+
+    public static INearestNeighbourModel Train(IClassificationProblemData problemData, int k, bool selfMatch = false, double[] weights = null) {
+      return new NearestNeighbourModel(problemData.Dataset,
+        problemData.TrainingIndices,
+        k,
+        selfMatch,
+        problemData.TargetVariable,
+        problemData.AllowedInputVariables,
+        weights,
+        problemData.ClassValues.ToArray());
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassificationSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassificationSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassificationSolution.cs	(revision 17041)
@@ -0,0 +1,53 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a nearest neighbour solution for a classification problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("NearestNeighbourClassificationSolution", "Represents a nearest neighbour solution for a classification problem which can be visualized in the GUI.")]
+  [StorableType("C336B1A1-A746-4DA9-A926-EC1CA4EFDB02")]
+  public sealed class NearestNeighbourClassificationSolution : ClassificationSolution, INearestNeighbourClassificationSolution {
+
+    public new INearestNeighbourModel Model {
+      get { return (INearestNeighbourModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private NearestNeighbourClassificationSolution(StorableConstructorFlag _) : base(_) { }
+    private NearestNeighbourClassificationSolution(NearestNeighbourClassificationSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NearestNeighbourClassificationSolution(INearestNeighbourModel nnModel, IClassificationProblemData problemData)
+      : base(nnModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NearestNeighbourClassificationSolution(this, cloner);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs	(revision 17041)
@@ -0,0 +1,444 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a nearest neighbour model for regression and classification
+  /// </summary>
+  [StorableType("A76C0823-3077-4ACE-8A40-E9B717C7DB60")]
+  [Item("NearestNeighbourModel", "Represents a nearest neighbour model for regression and classification.")]
+  public sealed class NearestNeighbourModel : ClassificationModel, INearestNeighbourModel {
+
+    private readonly object kdTreeLockObject = new object();
+
+    private alglib.nearestneighbor.kdtree kdTree;
+    public alglib.nearestneighbor.kdtree KDTree {
+      get { return kdTree; }
+      set {
+        if (value != kdTree) {
+          if (value == null) throw new ArgumentNullException();
+          kdTree = value;
+          OnChanged(EventArgs.Empty);
+        }
+      }
+    }
+
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return allowedInputVariables; }
+    }
+
+    [Storable]
+    private string[] allowedInputVariables;
+    [Storable]
+    private double[] classValues;
+    [Storable]
+    private int k;
+    [Storable(DefaultValue = false)]
+    private bool selfMatch;
+    [Storable(DefaultValue = null)]
+    private double[] weights; // not set for old versions loaded from disk
+    [Storable(DefaultValue = null)]
+    private double[] offsets; // not set for old versions loaded from disk
+
+    [StorableConstructor]
+    private NearestNeighbourModel(StorableConstructorFlag _) : base(_) {
+      kdTree = new alglib.nearestneighbor.kdtree();
+    }
+    private NearestNeighbourModel(NearestNeighbourModel original, Cloner cloner)
+      : base(original, cloner) {
+      kdTree = new alglib.nearestneighbor.kdtree();
+      kdTree.approxf = original.kdTree.approxf;
+      kdTree.boxmax = (double[])original.kdTree.boxmax.Clone();
+      kdTree.boxmin = (double[])original.kdTree.boxmin.Clone();
+      kdTree.buf = (double[])original.kdTree.buf.Clone();
+      kdTree.curboxmax = (double[])original.kdTree.curboxmax.Clone();
+      kdTree.curboxmin = (double[])original.kdTree.curboxmin.Clone();
+      kdTree.curdist = original.kdTree.curdist;
+      kdTree.debugcounter = original.kdTree.debugcounter;
+      kdTree.idx = (int[])original.kdTree.idx.Clone();
+      kdTree.kcur = original.kdTree.kcur;
+      kdTree.kneeded = original.kdTree.kneeded;
+      kdTree.n = original.kdTree.n;
+      kdTree.nodes = (int[])original.kdTree.nodes.Clone();
+      kdTree.normtype = original.kdTree.normtype;
+      kdTree.nx = original.kdTree.nx;
+      kdTree.ny = original.kdTree.ny;
+      kdTree.r = (double[])original.kdTree.r.Clone();
+      kdTree.rneeded = original.kdTree.rneeded;
+      kdTree.selfmatch = original.kdTree.selfmatch;
+      kdTree.splits = (double[])original.kdTree.splits.Clone();
+      kdTree.tags = (int[])original.kdTree.tags.Clone();
+      kdTree.x = (double[])original.kdTree.x.Clone();
+      kdTree.xy = (double[,])original.kdTree.xy.Clone();
+      selfMatch = original.selfMatch;
+      k = original.k;
+      isCompatibilityLoaded = original.IsCompatibilityLoaded;
+      if (!IsCompatibilityLoaded) {
+        weights = new double[original.weights.Length];
+        Array.Copy(original.weights, weights, weights.Length);
+        offsets = new double[original.offsets.Length];
+        Array.Copy(original.offsets, this.offsets, this.offsets.Length);
+      }
+      allowedInputVariables = (string[])original.allowedInputVariables.Clone();
+      if (original.classValues != null)
+        this.classValues = (double[])original.classValues.Clone();
+    }
+    public NearestNeighbourModel(IDataset dataset, IEnumerable<int> rows, int k, bool selfMatch, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<double> weights = null, double[] classValues = null)
+      : base(targetVariable) {
+      Name = ItemName;
+      Description = ItemDescription;
+      this.selfMatch = selfMatch;
+      this.k = k;
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+      double[,] inputMatrix;
+      if (IsCompatibilityLoaded) {
+        // no scaling
+        inputMatrix = dataset.ToArray(
+          this.allowedInputVariables.Concat(new string[] { targetVariable }),
+          rows);
+      } else {
+        this.offsets = this.allowedInputVariables
+          .Select(name => dataset.GetDoubleValues(name, rows).Average() * -1)
+          .Concat(new double[] { 0 }) // no offset for target variable
+          .ToArray();
+        if (weights == null) {
+          // automatic determination of weights (all features should have variance = 1)
+          this.weights = this.allowedInputVariables
+            .Select(name => {
+              var pop = dataset.GetDoubleValues(name, rows).StandardDeviationPop();
+              return pop.IsAlmost(0) ? 1.0 : 1.0 / pop;
+            })
+            .Concat(new double[] { 1.0 }) // no scaling for target variable
+            .ToArray();
+        } else {
+          // user specified weights (+ 1 for target)
+          this.weights = weights.Concat(new double[] { 1.0 }).ToArray();
+          if (this.weights.Length - 1 != this.allowedInputVariables.Length)
+            throw new ArgumentException("The number of elements in the weight vector must match the number of input variables");
+        }
+        inputMatrix = CreateScaledData(dataset, this.allowedInputVariables.Concat(new string[] { targetVariable }), rows, this.offsets, this.weights);
+      }
+
+      if (inputMatrix.ContainsNanOrInfinity())
+        throw new NotSupportedException(
+          "Nearest neighbour model does not support NaN or infinity values in the input dataset.");
+
+      this.kdTree = new alglib.nearestneighbor.kdtree();
+
+      var nRows = inputMatrix.GetLength(0);
+      var nFeatures = inputMatrix.GetLength(1) - 1;
+
+      if (classValues != null) {
+        this.classValues = (double[])classValues.Clone();
+        int nClasses = classValues.Length;
+        // map original class values to values [0..nClasses-1]
+        var classIndices = new Dictionary<double, double>();
+        for (int i = 0; i < nClasses; i++)
+          classIndices[classValues[i]] = i;
+
+        for (int row = 0; row < nRows; row++) {
+          inputMatrix[row, nFeatures] = classIndices[inputMatrix[row, nFeatures]];
+        }
+      }
+      alglib.nearestneighbor.kdtreebuild(inputMatrix, nRows, inputMatrix.GetLength(1) - 1, 1, 2, kdTree);
+    }
+
+    private static double[,] CreateScaledData(IDataset dataset, IEnumerable<string> variables, IEnumerable<int> rows, double[] offsets, double[] factors) {
+      var transforms =
+        variables.Select(
+          (_, colIdx) =>
+            new LinearTransformation(variables) { Addend = offsets[colIdx] * factors[colIdx], Multiplier = factors[colIdx] });
+      return dataset.ToArray(variables, transforms, rows);
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NearestNeighbourModel(this, cloner);
+    }
+
+    public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      double[,] inputData;
+      if (IsCompatibilityLoaded) {
+        inputData = dataset.ToArray(allowedInputVariables, rows);
+      } else {
+        inputData = CreateScaledData(dataset, allowedInputVariables, rows, offsets, weights);
+      }
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] dists = new double[k];
+      double[,] neighbours = new double[k, columns + 1];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        int numNeighbours;
+        lock (kdTreeLockObject) { // gkronber: the following calls change the kdTree data structure
+          numNeighbours = alglib.nearestneighbor.kdtreequeryknn(kdTree, x, k, selfMatch);
+          alglib.nearestneighbor.kdtreequeryresultsdistances(kdTree, ref dists);
+          alglib.nearestneighbor.kdtreequeryresultsxy(kdTree, ref neighbours);
+        }
+        if (selfMatch) {
+          // weights for neighbours are 1/d.
+          // override distances (=0) of exact matches using 1% of the distance of the next closest non-self-match neighbour -> selfmatches weight 100x more than the next closest neighbor.
+          // if all k neighbours are selfmatches then they all have weight 0.01.
+          double minDist = dists[0] + 1;
+          for (int i = 0; i < numNeighbours; i++) {
+            if ((minDist > dists[i]) && (dists[i] != 0)) {
+              minDist = dists[i];
+            }
+          }
+          minDist /= 100.0;
+          for (int i = 0; i < numNeighbours; i++) {
+            if (dists[i] == 0) {
+              dists[i] = minDist;
+            }
+          }
+        }
+        double distanceWeightedValue = 0.0;
+        double distsSum = 0.0;
+        for (int i = 0; i < numNeighbours; i++) {
+          distanceWeightedValue += neighbours[i, columns] / dists[i];
+          distsSum += 1.0 / dists[i];
+        }
+        yield return distanceWeightedValue / distsSum;
+      }
+    }
+
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+      if (classValues == null) throw new InvalidOperationException("No class values are defined.");
+      double[,] inputData;
+      if (IsCompatibilityLoaded) {
+        inputData = dataset.ToArray(allowedInputVariables, rows);
+      } else {
+        inputData = CreateScaledData(dataset, allowedInputVariables, rows, offsets, weights);
+      }
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      int[] y = new int[classValues.Length];
+      double[] dists = new double[k];
+      double[,] neighbours = new double[k, columns + 1];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        int numNeighbours;
+        lock (kdTreeLockObject) {
+          // gkronber: the following calls change the kdTree data structure
+          numNeighbours = alglib.nearestneighbor.kdtreequeryknn(kdTree, x, k, selfMatch);
+          alglib.nearestneighbor.kdtreequeryresultsdistances(kdTree, ref dists);
+          alglib.nearestneighbor.kdtreequeryresultsxy(kdTree, ref neighbours);
+        }
+        Array.Clear(y, 0, y.Length);
+        for (int i = 0; i < numNeighbours; i++) {
+          int classValue = (int)Math.Round(neighbours[i, columns]);
+          y[classValue]++;
+        }
+
+        // find class for with the largest probability value
+        int maxProbClassIndex = 0;
+        double maxProb = y[0];
+        for (int i = 1; i < y.Length; i++) {
+          if (maxProb < y[i]) {
+            maxProb = y[i];
+            maxProbClassIndex = i;
+          }
+        }
+        yield return classValues[maxProbClassIndex];
+      }
+    }
+
+
+    public bool IsProblemDataCompatible(IRegressionProblemData problemData, out string errorMessage) {
+      return RegressionModel.IsProblemDataCompatible(this, problemData, out errorMessage);
+    }
+
+    public override bool IsProblemDataCompatible(IDataAnalysisProblemData problemData, out string errorMessage) {
+      if (problemData == null) throw new ArgumentNullException("problemData", "The provided problemData is null.");
+
+      var regressionProblemData = problemData as IRegressionProblemData;
+      if (regressionProblemData != null)
+        return IsProblemDataCompatible(regressionProblemData, out errorMessage);
+
+      var classificationProblemData = problemData as IClassificationProblemData;
+      if (classificationProblemData != null)
+        return IsProblemDataCompatible(classificationProblemData, out errorMessage);
+
+      throw new ArgumentException("The problem data is not compatible with this nearest neighbour model. Instead a " + problemData.GetType().GetPrettyName() + " was provided.", "problemData");
+    }
+
+    IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new NearestNeighbourRegressionSolution(this, new RegressionProblemData(problemData));
+    }
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new NearestNeighbourClassificationSolution(this, new ClassificationProblemData(problemData));
+    }
+
+    #region events
+    public event EventHandler Changed;
+    private void OnChanged(EventArgs e) {
+      var handlers = Changed;
+      if (handlers != null)
+        handlers(this, e);
+    }
+    #endregion
+
+
+    // BackwardsCompatibility3.3
+    #region Backwards compatible code, remove with 3.4
+
+    private bool isCompatibilityLoaded = false; // new kNN models have the value false, kNN models loaded from disc have the value true
+    [Storable(DefaultValue = true)]
+    public bool IsCompatibilityLoaded {
+      get { return isCompatibilityLoaded; }
+      set { isCompatibilityLoaded = value; }
+    }
+    #endregion
+    #region persistence
+    [Storable]
+    public double KDTreeApproxF {
+      get { return kdTree.approxf; }
+      set { kdTree.approxf = value; }
+    }
+    [Storable]
+    public double[] KDTreeBoxMax {
+      get { return kdTree.boxmax; }
+      set { kdTree.boxmax = value; }
+    }
+    [Storable]
+    public double[] KDTreeBoxMin {
+      get { return kdTree.boxmin; }
+      set { kdTree.boxmin = value; }
+    }
+    [Storable]
+    public double[] KDTreeBuf {
+      get { return kdTree.buf; }
+      set { kdTree.buf = value; }
+    }
+    [Storable]
+    public double[] KDTreeCurBoxMax {
+      get { return kdTree.curboxmax; }
+      set { kdTree.curboxmax = value; }
+    }
+    [Storable]
+    public double[] KDTreeCurBoxMin {
+      get { return kdTree.curboxmin; }
+      set { kdTree.curboxmin = value; }
+    }
+    [Storable]
+    public double KDTreeCurDist {
+      get { return kdTree.curdist; }
+      set { kdTree.curdist = value; }
+    }
+    [Storable]
+    public int KDTreeDebugCounter {
+      get { return kdTree.debugcounter; }
+      set { kdTree.debugcounter = value; }
+    }
+    [Storable]
+    public int[] KDTreeIdx {
+      get { return kdTree.idx; }
+      set { kdTree.idx = value; }
+    }
+    [Storable]
+    public int KDTreeKCur {
+      get { return kdTree.kcur; }
+      set { kdTree.kcur = value; }
+    }
+    [Storable]
+    public int KDTreeKNeeded {
+      get { return kdTree.kneeded; }
+      set { kdTree.kneeded = value; }
+    }
+    [Storable]
+    public int KDTreeN {
+      get { return kdTree.n; }
+      set { kdTree.n = value; }
+    }
+    [Storable]
+    public int[] KDTreeNodes {
+      get { return kdTree.nodes; }
+      set { kdTree.nodes = value; }
+    }
+    [Storable]
+    public int KDTreeNormType {
+      get { return kdTree.normtype; }
+      set { kdTree.normtype = value; }
+    }
+    [Storable]
+    public int KDTreeNX {
+      get { return kdTree.nx; }
+      set { kdTree.nx = value; }
+    }
+    [Storable]
+    public int KDTreeNY {
+      get { return kdTree.ny; }
+      set { kdTree.ny = value; }
+    }
+    [Storable]
+    public double[] KDTreeR {
+      get { return kdTree.r; }
+      set { kdTree.r = value; }
+    }
+    [Storable]
+    public double KDTreeRNeeded {
+      get { return kdTree.rneeded; }
+      set { kdTree.rneeded = value; }
+    }
+    [Storable]
+    public bool KDTreeSelfMatch {
+      get { return kdTree.selfmatch; }
+      set { kdTree.selfmatch = value; }
+    }
+    [Storable]
+    public double[] KDTreeSplits {
+      get { return kdTree.splits; }
+      set { kdTree.splits = value; }
+    }
+    [Storable]
+    public int[] KDTreeTags {
+      get { return kdTree.tags; }
+      set { kdTree.tags = value; }
+    }
+    [Storable]
+    public double[] KDTreeX {
+      get { return kdTree.x; }
+      set { kdTree.x = value; }
+    }
+    [Storable]
+    public double[,] KDTreeXY {
+      get { return kdTree.xy; }
+      set { kdTree.xy = value; }
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs	(revision 17041)
@@ -0,0 +1,128 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Nearest neighbour regression data analysis algorithm.
+  /// </summary>
+  [Item("Nearest Neighbour Regression (kNN)", "Nearest neighbour regression data analysis algorithm (wrapper for ALGLIB).")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 150)]
+  [StorableType("3F940BE0-4F44-4F7F-A3EE-E47423C7F22D")]
+  public sealed class NearestNeighbourRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+    private const string KParameterName = "K";
+    private const string NearestNeighbourRegressionModelResultName = "Nearest neighbour regression solution";
+    private const string WeightsParameterName = "Weights";
+    private const string SelfMatchParameterName = "SelfMatch";
+
+    #region parameter properties
+    public IFixedValueParameter<IntValue> KParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[KParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> SelfMatchParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[SelfMatchParameterName]; }
+    }
+    public IValueParameter<DoubleArray> WeightsParameter {
+      get { return (IValueParameter<DoubleArray>)Parameters[WeightsParameterName]; }
+    }
+    #endregion
+    #region properties
+    public int K {
+      get { return KParameter.Value.Value; }
+      set {
+        if (value <= 0) throw new ArgumentException("K must be larger than zero.", "K");
+        else KParameter.Value.Value = value;
+      }
+    }
+    public bool SelfMatch {
+      get { return SelfMatchParameter.Value.Value; }
+      set { SelfMatchParameter.Value.Value = value; }
+    }
+    public DoubleArray Weights {
+      get { return WeightsParameter.Value; }
+      set { WeightsParameter.Value = value; }
+    }
+    #endregion
+
+    [StorableConstructor]
+    private NearestNeighbourRegression(StorableConstructorFlag _) : base(_) { }
+    private NearestNeighbourRegression(NearestNeighbourRegression original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NearestNeighbourRegression()
+      : base() {
+      Parameters.Add(new FixedValueParameter<IntValue>(KParameterName, "The number of nearest neighbours to consider for regression.", new IntValue(3)));
+      Parameters.Add(new OptionalValueParameter<DoubleArray>(WeightsParameterName, "Optional: use weights to specify individual scaling values for all features. If not set the weights are calculated automatically (each feature is scaled to unit variance)"));
+      Parameters.Add(new FixedValueParameter<BoolValue>(SelfMatchParameterName, "Should we use equal points for classification?", new BoolValue(false)));
+      Problem = new RegressionProblem();
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
+      if (!Parameters.ContainsKey(WeightsParameterName)) {
+        Parameters.Add(new OptionalValueParameter<DoubleArray>(WeightsParameterName, "Optional: use weights to specify individual scaling values for all features. If not set the weights are calculated automatically (each feature is scaled to unit variance)"));
+      }
+      if (!Parameters.ContainsKey(SelfMatchParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(SelfMatchParameterName, "Should we use equal points for classification?", new BoolValue(false)));
+      }
+      #endregion
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NearestNeighbourRegression(this, cloner);
+    }
+
+    #region nearest neighbour
+    protected override void Run(CancellationToken cancellationToken) {
+      double[] weights = null;
+      if (Weights != null) weights = Weights.CloneAsArray();
+      var solution = CreateNearestNeighbourRegressionSolution(Problem.ProblemData, K, SelfMatch, weights);
+      Results.Add(new Result(NearestNeighbourRegressionModelResultName, "The nearest neighbour regression solution.", solution));
+    }
+
+    public static IRegressionSolution CreateNearestNeighbourRegressionSolution(IRegressionProblemData problemData, int k, bool selfMatch = false, double[] weights = null) {
+      var clonedProblemData = (IRegressionProblemData)problemData.Clone();
+      return new NearestNeighbourRegressionSolution(Train(problemData, k, selfMatch, weights), clonedProblemData);
+    }
+
+    public static INearestNeighbourModel Train(IRegressionProblemData problemData, int k, bool selfMatch = false, double[] weights = null) {
+      return new NearestNeighbourModel(problemData.Dataset,
+        problemData.TrainingIndices,
+        k,
+        selfMatch,
+        problemData.TargetVariable,
+        problemData.AllowedInputVariables,
+        weights);
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegressionSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegressionSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegressionSolution.cs	(revision 17041)
@@ -0,0 +1,53 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a nearest neighbour solution for a regression problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("NearestNeighbourRegressionSolution", "Represents a nearest neighbour solution for a regression problem which can be visualized in the GUI.")]
+  [StorableType("71C86C1A-C917-4CA4-A239-C167C1C793F8")]
+  public sealed class NearestNeighbourRegressionSolution : RegressionSolution, INearestNeighbourRegressionSolution {
+
+    public new INearestNeighbourModel Model {
+      get { return (INearestNeighbourModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private NearestNeighbourRegressionSolution(StorableConstructorFlag _) : base(_) { }
+    private NearestNeighbourRegressionSolution(NearestNeighbourRegressionSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NearestNeighbourRegressionSolution(INearestNeighbourModel nnModel, IRegressionProblemData problemData)
+      : base(nnModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NearestNeighbourRegressionSolution(this, cloner);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs	(revision 17041)
@@ -0,0 +1,227 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Neural network classification data analysis algorithm.
+  /// </summary>
+  [Item("Neural Network Classification (NN)", "Neural network classification data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/neuralnetworks.php")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 130)]
+  [StorableType("8D44A55B-B162-42D6-A761-A420F4D107C4")]
+  public sealed class NeuralNetworkClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+    private const string DecayParameterName = "Decay";
+    private const string HiddenLayersParameterName = "HiddenLayers";
+    private const string NodesInFirstHiddenLayerParameterName = "NodesInFirstHiddenLayer";
+    private const string NodesInSecondHiddenLayerParameterName = "NodesInSecondHiddenLayer";
+    private const string RestartsParameterName = "Restarts";
+    private const string NeuralNetworkClassificationModelResultName = "Neural network classification solution";
+
+    #region parameter properties
+    public IFixedValueParameter<DoubleValue> DecayParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[DecayParameterName]; }
+    }
+    public IConstrainedValueParameter<IntValue> HiddenLayersParameter {
+      get { return (IConstrainedValueParameter<IntValue>)Parameters[HiddenLayersParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInFirstHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInFirstHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInSecondHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInSecondHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> RestartsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[RestartsParameterName]; }
+    }
+    #endregion
+
+    #region properties
+    public double Decay {
+      get { return DecayParameter.Value.Value; }
+      set {
+        if (value < 0.001 || value > 100) throw new ArgumentException("The decay parameter should be set to a value between 0.001 and 100.", "Decay");
+        DecayParameter.Value.Value = value;
+      }
+    }
+    public int HiddenLayers {
+      get { return HiddenLayersParameter.Value.Value; }
+      set {
+        if (value < 0 || value > 2) throw new ArgumentException("The number of hidden layers should be set to 0, 1, or 2.", "HiddenLayers");
+        HiddenLayersParameter.Value = (from v in HiddenLayersParameter.ValidValues
+                                       where v.Value == value
+                                       select v)
+                                      .Single();
+      }
+    }
+    public int NodesInFirstHiddenLayer {
+      get { return NodesInFirstHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first hidden layer must be at least one.", "NodesInFirstHiddenLayer");
+        NodesInFirstHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int NodesInSecondHiddenLayer {
+      get { return NodesInSecondHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first second layer must be at least one.", "NodesInSecondHiddenLayer");
+        NodesInSecondHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int Restarts {
+      get { return RestartsParameter.Value.Value; }
+      set {
+        if (value < 0) throw new ArgumentException("The number of restarts must be positive.", "Restarts");
+        RestartsParameter.Value.Value = value;
+      }
+    }
+    #endregion
+
+
+    [StorableConstructor]
+    private NeuralNetworkClassification(StorableConstructorFlag _) : base(_) { }
+    private NeuralNetworkClassification(NeuralNetworkClassification original, Cloner cloner)
+      : base(original, cloner) {
+      RegisterEventHandlers();
+    }
+    public NeuralNetworkClassification()
+      : base() {
+      var validHiddenLayerValues = new ItemSet<IntValue>(new IntValue[] {
+        (IntValue)new IntValue(0).AsReadOnly(),
+        (IntValue)new IntValue(1).AsReadOnly(),
+        (IntValue)new IntValue(2).AsReadOnly() });
+      var selectedHiddenLayerValue = (from v in validHiddenLayerValues
+                                      where v.Value == 1
+                                      select v)
+                                     .Single();
+      Parameters.Add(new FixedValueParameter<DoubleValue>(DecayParameterName, "The decay parameter for the training phase of the neural network. This parameter determines the strengh of regularization and should be set to a value between 0.001 (weak regularization) to 100 (very strong regularization). The correct value should be determined via cross-validation.", new DoubleValue(1)));
+      Parameters.Add(new ConstrainedValueParameter<IntValue>(HiddenLayersParameterName, "The number of hidden layers for the neural network (0, 1, or 2)", validHiddenLayerValues, selectedHiddenLayerValue));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInFirstHiddenLayerParameterName, "The number of nodes in the first hidden layer. This value is not used if the number of hidden layers is zero.", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInSecondHiddenLayerParameterName, "The number of nodes in the second hidden layer. This value is not used if the number of hidden layers is zero or one.", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of restarts for learning.", new IntValue(2)));
+
+      RestartsParameter.Hidden = true;
+      NodesInSecondHiddenLayerParameter.Hidden = true;
+
+      RegisterEventHandlers();
+
+      Problem = new ClassificationProblem();
+    }
+
+    private void RegisterEventHandlers() {
+      HiddenLayersParameter.Value.ValueChanged += HiddenLayersParameterValueValueChanged;
+      HiddenLayersParameter.ValueChanged += HiddenLayersParameterValueChanged;
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      RegisterEventHandlers();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkClassification(this, cloner);
+    }
+    private void HiddenLayersParameterValueChanged(object source, EventArgs e) {
+      HiddenLayersParameter.Value.ValueChanged += HiddenLayersParameterValueValueChanged;
+      HiddenLayersParameterValueValueChanged(this, EventArgs.Empty);
+    }
+
+    private void HiddenLayersParameterValueValueChanged(object source, EventArgs e) {
+      if (HiddenLayers == 0) {
+        NodesInFirstHiddenLayerParameter.Hidden = true;
+        NodesInSecondHiddenLayerParameter.Hidden = true;
+      } else if (HiddenLayers == 1) {
+        NodesInFirstHiddenLayerParameter.Hidden = false;
+        NodesInSecondHiddenLayerParameter.Hidden = true;
+      } else {
+        NodesInFirstHiddenLayerParameter.Hidden = false;
+        NodesInSecondHiddenLayerParameter.Hidden = false;
+      }
+    }
+
+    #region neural network
+    protected override void Run(CancellationToken cancellationToken) {
+      double rmsError, avgRelError, relClassError;
+      var solution = CreateNeuralNetworkClassificationSolution(Problem.ProblemData, HiddenLayers, NodesInFirstHiddenLayer, NodesInSecondHiddenLayer, Decay, Restarts, out rmsError, out avgRelError, out relClassError);
+      Results.Add(new Result(NeuralNetworkClassificationModelResultName, "The neural network classification solution.", solution));
+      Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the neural network classification solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Average relative error", "The average of relative errors of the neural network classification solution on the training set.", new PercentValue(avgRelError)));
+      Results.Add(new Result("Relative classification error", "The percentage of misclassified samples.", new PercentValue(relClassError)));
+    }
+
+    public static IClassificationSolution CreateNeuralNetworkClassificationSolution(IClassificationProblemData problemData, int nLayers, int nHiddenNodes1, int nHiddenNodes2, double decay, int restarts,
+      out double rmsError, out double avgRelError, out double relClassError) {
+      var dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+      double[,] inputMatrix = dataset.ToArray(allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      if (inputMatrix.ContainsNanOrInfinity())
+        throw new NotSupportedException("Neural network classification does not support NaN or infinity values in the input dataset.");
+
+      int nRows = inputMatrix.GetLength(0);
+      int nFeatures = inputMatrix.GetLength(1) - 1;
+      double[] classValues = dataset.GetDoubleValues(targetVariable).Distinct().OrderBy(x => x).ToArray();
+      int nClasses = classValues.Count();
+      // map original class values to values [0..nClasses-1]
+      Dictionary<double, double> classIndices = new Dictionary<double, double>();
+      for (int i = 0; i < nClasses; i++) {
+        classIndices[classValues[i]] = i;
+      }
+      for (int row = 0; row < nRows; row++) {
+        inputMatrix[row, nFeatures] = classIndices[inputMatrix[row, nFeatures]];
+      }
+
+      alglib.multilayerperceptron multiLayerPerceptron = null;
+      if (nLayers == 0) {
+        alglib.mlpcreatec0(allowedInputVariables.Count(), nClasses, out multiLayerPerceptron);
+      } else if (nLayers == 1) {
+        alglib.mlpcreatec1(allowedInputVariables.Count(), nHiddenNodes1, nClasses, out multiLayerPerceptron);
+      } else if (nLayers == 2) {
+        alglib.mlpcreatec2(allowedInputVariables.Count(), nHiddenNodes1, nHiddenNodes2, nClasses, out multiLayerPerceptron);
+      } else throw new ArgumentException("Number of layers must be zero, one, or two.", "nLayers");
+      alglib.mlpreport rep;
+
+      int info;
+      // using mlptrainlm instead of mlptraines or mlptrainbfgs because only one parameter is necessary
+      alglib.mlptrainlm(multiLayerPerceptron, inputMatrix, nRows, decay, restarts, out info, out rep);
+      if (info != 2) throw new ArgumentException("Error in calculation of neural network classification solution");
+
+      rmsError = alglib.mlprmserror(multiLayerPerceptron, inputMatrix, nRows);
+      avgRelError = alglib.mlpavgrelerror(multiLayerPerceptron, inputMatrix, nRows);
+      relClassError = alglib.mlpclserror(multiLayerPerceptron, inputMatrix, nRows) / (double)nRows;
+
+      var problemDataClone = (IClassificationProblemData)problemData.Clone();
+      return new NeuralNetworkClassificationSolution(new NeuralNetworkModel(multiLayerPerceptron, targetVariable, allowedInputVariables, problemDataClone.ClassValues.ToArray()), problemDataClone);
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassificationSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassificationSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassificationSolution.cs	(revision 17041)
@@ -0,0 +1,54 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a neural network solution for a classification problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("NeuralNetworkClassificationSolution", "Represents a neural network solution for a classification problem which can be visualized in the GUI.")]
+  [StorableType("7B08B27F-66B6-47C8-946E-51B28F603B43")]
+  public sealed class NeuralNetworkClassificationSolution : ClassificationSolution, INeuralNetworkClassificationSolution {
+
+    public new INeuralNetworkModel Model {
+      get { return (INeuralNetworkModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private NeuralNetworkClassificationSolution(StorableConstructorFlag _) : base(_) { }
+    private NeuralNetworkClassificationSolution(NeuralNetworkClassificationSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NeuralNetworkClassificationSolution(INeuralNetworkModel nnModel, IClassificationProblemData problemData)
+      : base(nnModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkClassificationSolution(this, cloner);
+    }
+
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassification.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassification.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassification.cs	(revision 17041)
@@ -0,0 +1,211 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Neural network ensemble classification data analysis algorithm.
+  /// </summary>
+  [Item("Neural Network Ensemble Classification (NN)", "Neural network ensemble classification data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/mlpensembles.php")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 140)]
+  [StorableType("21B48D73-B907-4710-854A-C549F8C66CFF")]
+  public sealed class NeuralNetworkEnsembleClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+    private const string EnsembleSizeParameterName = "EnsembleSize";
+    private const string DecayParameterName = "Decay";
+    private const string HiddenLayersParameterName = "HiddenLayers";
+    private const string NodesInFirstHiddenLayerParameterName = "NodesInFirstHiddenLayer";
+    private const string NodesInSecondHiddenLayerParameterName = "NodesInSecondHiddenLayer";
+    private const string RestartsParameterName = "Restarts";
+    private const string NeuralNetworkEnsembleClassificationModelResultName = "Neural network ensemble classification solution";
+
+    #region parameter properties
+    public IFixedValueParameter<IntValue> EnsembleSizeParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[EnsembleSizeParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> DecayParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[DecayParameterName]; }
+    }
+    public IConstrainedValueParameter<IntValue> HiddenLayersParameter {
+      get { return (IConstrainedValueParameter<IntValue>)Parameters[HiddenLayersParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInFirstHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInFirstHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInSecondHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInSecondHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> RestartsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[RestartsParameterName]; }
+    }
+    #endregion
+
+    #region properties
+    public int EnsembleSize {
+      get { return EnsembleSizeParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of models in the ensemble must be positive and at least one.", "EnsembleSize");
+        EnsembleSizeParameter.Value.Value = value;
+      }
+    }
+    public double Decay {
+      get { return DecayParameter.Value.Value; }
+      set {
+        if (value < 0.001 || value > 100) throw new ArgumentException("The decay parameter should be set to a value between 0.001 and 100.", "Decay");
+        DecayParameter.Value.Value = value;
+      }
+    }
+    public int HiddenLayers {
+      get { return HiddenLayersParameter.Value.Value; }
+      set {
+        if (value < 0 || value > 2) throw new ArgumentException("The number of hidden layers should be set to 0, 1, or 2.", "HiddenLayers");
+        HiddenLayersParameter.Value = (from v in HiddenLayersParameter.ValidValues
+                                       where v.Value == value
+                                       select v)
+                                      .Single();
+      }
+    }
+    public int NodesInFirstHiddenLayer {
+      get { return NodesInFirstHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first hidden layer must be at least one.", "NodesInFirstHiddenLayer");
+        NodesInFirstHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int NodesInSecondHiddenLayer {
+      get { return NodesInSecondHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first second layer must be at least one.", "NodesInSecondHiddenLayer");
+        NodesInSecondHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int Restarts {
+      get { return RestartsParameter.Value.Value; }
+      set {
+        if (value < 0) throw new ArgumentException("The number of restarts must be positive.", "Restarts");
+        RestartsParameter.Value.Value = value;
+      }
+    }
+    #endregion
+
+
+    [StorableConstructor]
+    private NeuralNetworkEnsembleClassification(StorableConstructorFlag _) : base(_) { }
+    private NeuralNetworkEnsembleClassification(NeuralNetworkEnsembleClassification original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NeuralNetworkEnsembleClassification()
+      : base() {
+      var validHiddenLayerValues = new ItemSet<IntValue>(new IntValue[] {
+        (IntValue)new IntValue(0).AsReadOnly(),
+        (IntValue)new IntValue(1).AsReadOnly(),
+        (IntValue)new IntValue(2).AsReadOnly() });
+      var selectedHiddenLayerValue = (from v in validHiddenLayerValues
+                                      where v.Value == 1
+                                      select v)
+                                     .Single();
+      Parameters.Add(new FixedValueParameter<IntValue>(EnsembleSizeParameterName, "The number of simple neural network models in the ensemble. A good value is 10.", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(DecayParameterName, "The decay parameter for the training phase of the neural network. This parameter determines the strengh of regularization and should be set to a value between 0.001 (weak regularization) to 100 (very strong regularization). The correct value should be determined via cross-validation.", new DoubleValue(0.001)));
+      Parameters.Add(new ConstrainedValueParameter<IntValue>(HiddenLayersParameterName, "The number of hidden layers for the neural network (0, 1, or 2)", validHiddenLayerValues, selectedHiddenLayerValue));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInFirstHiddenLayerParameterName, "The number of nodes in the first hidden layer. The value should be rather large (30-100 nodes) in order to make the network highly flexible and run into the early stopping criterion). This value is not used if the number of hidden layers is zero.", new IntValue(100)));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInSecondHiddenLayerParameterName, "The number of nodes in the second hidden layer. This value is not used if the number of hidden layers is zero or one.", new IntValue(100)));
+      Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of restarts for learning.", new IntValue(2)));
+
+      HiddenLayersParameter.Hidden = true;
+      NodesInFirstHiddenLayerParameter.Hidden = true;
+      NodesInSecondHiddenLayerParameter.Hidden = true;
+      RestartsParameter.Hidden = true;
+
+      Problem = new ClassificationProblem();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkEnsembleClassification(this, cloner);
+    }
+
+    #region neural network ensemble
+    protected override void Run(CancellationToken cancellationToken) {
+      double rmsError, avgRelError, relClassError;
+      var solution = CreateNeuralNetworkEnsembleClassificationSolution(Problem.ProblemData, EnsembleSize, HiddenLayers, NodesInFirstHiddenLayer, NodesInSecondHiddenLayer, Decay, Restarts, out rmsError, out avgRelError, out relClassError);
+      Results.Add(new Result(NeuralNetworkEnsembleClassificationModelResultName, "The neural network ensemble classification solution.", solution));
+      Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the neural network ensemble classification solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Average relative error", "The average of relative errors of the neural network ensemble classification solution on the training set.", new PercentValue(avgRelError)));
+      Results.Add(new Result("Relative classification error", "The percentage of misclassified samples.", new PercentValue(relClassError)));
+    }
+
+    public static IClassificationSolution CreateNeuralNetworkEnsembleClassificationSolution(IClassificationProblemData problemData, int ensembleSize, int nLayers, int nHiddenNodes1, int nHiddenNodes2, double decay, int restarts,
+      out double rmsError, out double avgRelError, out double relClassError) {
+      var dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+      double[,] inputMatrix = dataset.ToArray(allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      if (inputMatrix.ContainsNanOrInfinity())
+        throw new NotSupportedException("Neural network ensemble classification does not support NaN or infinity values in the input dataset.");
+
+      int nRows = inputMatrix.GetLength(0);
+      int nFeatures = inputMatrix.GetLength(1) - 1;
+      double[] classValues = dataset.GetDoubleValues(targetVariable).Distinct().OrderBy(x => x).ToArray();
+      int nClasses = classValues.Count();
+      // map original class values to values [0..nClasses-1]
+      Dictionary<double, double> classIndices = new Dictionary<double, double>();
+      for (int i = 0; i < nClasses; i++) {
+        classIndices[classValues[i]] = i;
+      }
+      for (int row = 0; row < nRows; row++) {
+        inputMatrix[row, nFeatures] = classIndices[inputMatrix[row, nFeatures]];
+      }
+
+      alglib.mlpensemble mlpEnsemble = null;
+      if (nLayers == 0) {
+        alglib.mlpecreatec0(allowedInputVariables.Count(), nClasses, ensembleSize, out mlpEnsemble);
+      } else if (nLayers == 1) {
+        alglib.mlpecreatec1(allowedInputVariables.Count(), nHiddenNodes1, nClasses, ensembleSize, out mlpEnsemble);
+      } else if (nLayers == 2) {
+        alglib.mlpecreatec2(allowedInputVariables.Count(), nHiddenNodes1, nHiddenNodes2, nClasses, ensembleSize, out mlpEnsemble);
+      } else throw new ArgumentException("Number of layers must be zero, one, or two.", "nLayers");
+      alglib.mlpreport rep;
+
+      int info;
+      alglib.mlpetraines(mlpEnsemble, inputMatrix, nRows, decay, restarts, out info, out rep);
+      if (info != 6) throw new ArgumentException("Error in calculation of neural network ensemble classification solution");
+
+      rmsError = alglib.mlpermserror(mlpEnsemble, inputMatrix, nRows);
+      avgRelError = alglib.mlpeavgrelerror(mlpEnsemble, inputMatrix, nRows);
+      relClassError = alglib.mlperelclserror(mlpEnsemble, inputMatrix, nRows);
+      var problemDataClone = (IClassificationProblemData)problemData.Clone();
+      return new NeuralNetworkEnsembleClassificationSolution(new NeuralNetworkEnsembleModel(mlpEnsemble, targetVariable, allowedInputVariables, problemDataClone.ClassValues.ToArray()), problemDataClone);
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassificationSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassificationSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassificationSolution.cs	(revision 17041)
@@ -0,0 +1,53 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a neural network ensemble solution for a classification problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("NeuralNetworkEnsembleClassificationSolution", "Represents a neural network ensemble solution for a classification problem which can be visualized in the GUI.")]
+  [StorableType("7D834C91-2790-44B6-97A4-BC7409D869FD")]
+  public sealed class NeuralNetworkEnsembleClassificationSolution : ClassificationSolution, INeuralNetworkEnsembleClassificationSolution {
+
+    public new INeuralNetworkEnsembleModel Model {
+      get { return (INeuralNetworkEnsembleModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private NeuralNetworkEnsembleClassificationSolution(StorableConstructorFlag _) : base(_) { }
+    private NeuralNetworkEnsembleClassificationSolution(NeuralNetworkEnsembleClassificationSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NeuralNetworkEnsembleClassificationSolution(INeuralNetworkEnsembleModel nnModel, IClassificationProblemData problemData)
+      : base(nnModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkEnsembleClassificationSolution(this, cloner);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleModel.cs	(revision 17041)
@@ -0,0 +1,238 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a neural network ensembel model for regression and classification
+  /// </summary>
+  [StorableType("51B29670-27BD-405C-A521-39814E4BD857")]
+  [Item("NeuralNetworkEnsembleModel", "Represents a neural network ensemble for regression and classification.")]
+  public sealed class NeuralNetworkEnsembleModel : ClassificationModel, INeuralNetworkEnsembleModel {
+
+    private object mlpEnsembleLocker = new object();
+    private alglib.mlpensemble mlpEnsemble;
+
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return allowedInputVariables; }
+    }
+
+    [Storable]
+    private string targetVariable;
+    [Storable]
+    private string[] allowedInputVariables;
+    [Storable]
+    private double[] classValues;
+    [StorableConstructor]
+    private NeuralNetworkEnsembleModel(StorableConstructorFlag _) : base(_) {
+      mlpEnsemble = new alglib.mlpensemble();
+    }
+    private NeuralNetworkEnsembleModel(NeuralNetworkEnsembleModel original, Cloner cloner)
+      : base(original, cloner) {
+      mlpEnsemble = new alglib.mlpensemble();
+      string serializedEnsemble;
+      alglib.mlpeserialize(original.mlpEnsemble, out serializedEnsemble);
+      alglib.mlpeunserialize(serializedEnsemble, out this.mlpEnsemble);
+      targetVariable = original.targetVariable;
+      allowedInputVariables = (string[])original.allowedInputVariables.Clone();
+      if (original.classValues != null)
+        this.classValues = (double[])original.classValues.Clone();
+    }
+    public NeuralNetworkEnsembleModel(alglib.mlpensemble mlpEnsemble, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues = null)
+      : base(targetVariable) {
+      this.name = ItemName;
+      this.description = ItemDescription;
+      this.mlpEnsemble = mlpEnsemble;
+      this.targetVariable = targetVariable;
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+      if (classValues != null)
+        this.classValues = (double[])classValues.Clone();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkEnsembleModel(this, cloner);
+    }
+
+    public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = dataset.ToArray(allowedInputVariables, rows);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] y = new double[1];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        // mlpeprocess writes data in mlpEnsemble and is therefore not thread-safe
+        lock (mlpEnsembleLocker) {
+          alglib.mlpeprocess(mlpEnsemble, x, ref y);
+        }
+        yield return y[0];
+      }
+    }
+
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = dataset.ToArray(allowedInputVariables, rows);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] y = new double[classValues.Length];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        // mlpeprocess writes data in mlpEnsemble and is therefore not thread-safe
+        lock (mlpEnsembleLocker) {
+          alglib.mlpeprocess(mlpEnsemble, x, ref y);
+        }
+        // find class for with the largest probability value
+        int maxProbClassIndex = 0;
+        double maxProb = y[0];
+        for (int i = 1; i < y.Length; i++) {
+          if (maxProb < y[i]) {
+            maxProb = y[i];
+            maxProbClassIndex = i;
+          }
+        }
+        yield return classValues[maxProbClassIndex];
+      }
+    }
+
+
+    public bool IsProblemDataCompatible(IRegressionProblemData problemData, out string errorMessage) {
+      return RegressionModel.IsProblemDataCompatible(this, problemData, out errorMessage);
+    }
+
+    public override bool IsProblemDataCompatible(IDataAnalysisProblemData problemData, out string errorMessage) {
+      if (problemData == null) throw new ArgumentNullException("problemData", "The provided problemData is null.");
+
+      var regressionProblemData = problemData as IRegressionProblemData;
+      if (regressionProblemData != null)
+        return IsProblemDataCompatible(regressionProblemData, out errorMessage);
+
+      var classificationProblemData = problemData as IClassificationProblemData;
+      if (classificationProblemData != null)
+        return IsProblemDataCompatible(classificationProblemData, out errorMessage);
+
+      throw new ArgumentException("The problem data is not compatible with this neural network ensemble. Instead a " + problemData.GetType().GetPrettyName() + " was provided.", "problemData");
+    }
+
+    public IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new NeuralNetworkEnsembleRegressionSolution(this, new RegressionEnsembleProblemData(problemData));
+    }
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new NeuralNetworkEnsembleClassificationSolution(this, new ClassificationEnsembleProblemData(problemData));
+    }
+
+    #region persistence
+    [Storable]
+    private string MultiLayerPerceptronEnsembleNetwork {
+      get {
+        string serializedNetwork;
+        alglib.mlpeserialize(this.mlpEnsemble, out serializedNetwork);
+        return serializedNetwork;
+      }
+      set {
+        alglib.mlpeunserialize(value, out this.mlpEnsemble);
+      }
+    }
+
+    [Storable]
+    private double[] MultiLayerPerceptronEnsembleColumnMeans {
+      get { return mlpEnsemble.innerobj.columnmeans; }
+      set {
+        mlpEnsemble.innerobj.columnmeans = value;
+        mlpEnsemble.innerobj.network.columnmeans = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronEnsembleColumnSigmas {
+      get { return mlpEnsemble.innerobj.columnsigmas; }
+      set {
+        mlpEnsemble.innerobj.columnsigmas = value;
+        mlpEnsemble.innerobj.network.columnsigmas = value;
+      }
+    }
+    [Storable(OldName = "MultiLayerPerceptronEnsembleDfdnet")]
+    private double[] MultiLayerPerceptronEnsembleDfdnet {
+      set {
+        mlpEnsemble.innerobj.network.dfdnet = value;
+      }
+    }
+    [Storable]
+    private int MultiLayerPerceptronEnsembleSize {
+      get { return mlpEnsemble.innerobj.ensemblesize; }
+      set {
+        mlpEnsemble.innerobj.ensemblesize = value;
+        mlpEnsemble.innerobj.ensemblesize = value;
+      }
+    }
+    [Storable(OldName = "MultiLayerPerceptronEnsembleNeurons")]
+    private double[] MultiLayerPerceptronEnsembleNeurons {
+      set { mlpEnsemble.innerobj.network.neurons = value; }
+    }
+    [Storable(OldName = "MultiLayerPerceptronEnsembleSerializedMlp")]
+    private double[] MultiLayerPerceptronEnsembleSerializedMlp {
+      set {
+        mlpEnsemble.innerobj.network.dfdnet = value;
+      }
+    }
+    [Storable(OldName = "MultiLayerPerceptronStuctinfo")]
+    private int[] MultiLayerPerceptronStuctinfo {
+      set {
+        mlpEnsemble.innerobj.network.structinfo = value;
+      }
+    }
+
+    [Storable]
+    private double[] MultiLayerPerceptronWeights {
+      get {
+        return mlpEnsemble.innerobj.weights;
+      }
+      set {
+        mlpEnsemble.innerobj.weights = value;
+        mlpEnsemble.innerobj.network.weights = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronY {
+      get {
+        return mlpEnsemble.innerobj.y;
+      }
+      set {
+        mlpEnsemble.innerobj.y = value;
+        mlpEnsemble.innerobj.network.y = value;
+      }
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegression.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegression.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegression.cs	(revision 17041)
@@ -0,0 +1,197 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Neural network ensemble regression data analysis algorithm.
+  /// </summary>
+  [Item("Neural Network Ensemble Regression (NN)", "Neural network ensemble regression data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/mlpensembles.php")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 140)]
+  [StorableType("FD50BD78-3586-4C31-87AE-5490E717F4E7")]
+  public sealed class NeuralNetworkEnsembleRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+    private const string EnsembleSizeParameterName = "EnsembleSize";
+    private const string DecayParameterName = "Decay";
+    private const string HiddenLayersParameterName = "HiddenLayers";
+    private const string NodesInFirstHiddenLayerParameterName = "NodesInFirstHiddenLayer";
+    private const string NodesInSecondHiddenLayerParameterName = "NodesInSecondHiddenLayer";
+    private const string RestartsParameterName = "Restarts";
+    private const string NeuralNetworkEnsembleRegressionModelResultName = "Neural network ensemble regression solution";
+
+    #region parameter properties
+    public IFixedValueParameter<IntValue> EnsembleSizeParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[EnsembleSizeParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> DecayParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[DecayParameterName]; }
+    }
+    public IConstrainedValueParameter<IntValue> HiddenLayersParameter {
+      get { return (IConstrainedValueParameter<IntValue>)Parameters[HiddenLayersParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInFirstHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInFirstHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInSecondHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInSecondHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> RestartsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[RestartsParameterName]; }
+    }
+    #endregion
+
+    #region properties
+    public int EnsembleSize {
+      get { return EnsembleSizeParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of models in the ensemble must be positive and at least one.", "EnsembleSize");
+        EnsembleSizeParameter.Value.Value = value;
+      }
+    }
+    public double Decay {
+      get { return DecayParameter.Value.Value; }
+      set {
+        if (value < 0.001 || value > 100) throw new ArgumentException("The decay parameter should be set to a value between 0.001 and 100.", "Decay");
+        DecayParameter.Value.Value = value;
+      }
+    }
+    public int HiddenLayers {
+      get { return HiddenLayersParameter.Value.Value; }
+      set {
+        if (value < 0 || value > 2) throw new ArgumentException("The number of hidden layers should be set to 0, 1, or 2.", "HiddenLayers");
+        HiddenLayersParameter.Value = (from v in HiddenLayersParameter.ValidValues
+                                       where v.Value == value
+                                       select v)
+                                      .Single();
+      }
+    }
+    public int NodesInFirstHiddenLayer {
+      get { return NodesInFirstHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first hidden layer must be at least one.", "NodesInFirstHiddenLayer");
+        NodesInFirstHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int NodesInSecondHiddenLayer {
+      get { return NodesInSecondHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first second layer must be at least one.", "NodesInSecondHiddenLayer");
+        NodesInSecondHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int Restarts {
+      get { return RestartsParameter.Value.Value; }
+      set {
+        if (value < 0) throw new ArgumentException("The number of restarts must be positive.", "Restarts");
+        RestartsParameter.Value.Value = value;
+      }
+    }
+    #endregion
+
+
+    [StorableConstructor]
+    private NeuralNetworkEnsembleRegression(StorableConstructorFlag _) : base(_) { }
+    private NeuralNetworkEnsembleRegression(NeuralNetworkEnsembleRegression original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NeuralNetworkEnsembleRegression()
+      : base() {
+      var validHiddenLayerValues = new ItemSet<IntValue>(new IntValue[] {
+        (IntValue)new IntValue(0).AsReadOnly(),
+        (IntValue)new IntValue(1).AsReadOnly(),
+        (IntValue)new IntValue(2).AsReadOnly() });
+      var selectedHiddenLayerValue = (from v in validHiddenLayerValues
+                                      where v.Value == 1
+                                      select v)
+                                     .Single();
+      Parameters.Add(new FixedValueParameter<IntValue>(EnsembleSizeParameterName, "The number of simple neural network models in the ensemble. A good value is 10.", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(DecayParameterName, "The decay parameter for the training phase of the neural network. This parameter determines the strengh of regularization and should be set to a value between 0.001 (weak regularization) to 100 (very strong regularization). The correct value should be determined via cross-validation.", new DoubleValue(0.001)));
+      Parameters.Add(new ConstrainedValueParameter<IntValue>(HiddenLayersParameterName, "The number of hidden layers for the neural network (0, 1, or 2)", validHiddenLayerValues, selectedHiddenLayerValue));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInFirstHiddenLayerParameterName, "The number of nodes in the first hidden layer. The value should be rather large (30-100 nodes) in order to make the network highly flexible and run into the early stopping criterion). This value is not used if the number of hidden layers is zero.", new IntValue(100)));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInSecondHiddenLayerParameterName, "The number of nodes in the second hidden layer. This value is not used if the number of hidden layers is zero or one.", new IntValue(100)));
+      Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of restarts for learning.", new IntValue(2)));
+
+      HiddenLayersParameter.Hidden = true;
+      NodesInFirstHiddenLayerParameter.Hidden = true;
+      NodesInSecondHiddenLayerParameter.Hidden = true;
+      RestartsParameter.Hidden = true;
+
+      Problem = new RegressionProblem();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkEnsembleRegression(this, cloner);
+    }
+
+    #region neural network ensemble
+    protected override void Run(CancellationToken cancellationToken) {
+      double rmsError, avgRelError;
+      var solution = CreateNeuralNetworkEnsembleRegressionSolution(Problem.ProblemData, EnsembleSize, HiddenLayers, NodesInFirstHiddenLayer, NodesInSecondHiddenLayer, Decay, Restarts, out rmsError, out avgRelError);
+      Results.Add(new Result(NeuralNetworkEnsembleRegressionModelResultName, "The neural network ensemble regression solution.", solution));
+      Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the neural network ensemble regression solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Average relative error", "The average of relative errors of the neural network ensemble regression solution on the training set.", new PercentValue(avgRelError)));
+    }
+
+    public static IRegressionSolution CreateNeuralNetworkEnsembleRegressionSolution(IRegressionProblemData problemData, int ensembleSize, int nLayers, int nHiddenNodes1, int nHiddenNodes2, double decay, int restarts,
+      out double rmsError, out double avgRelError) {
+      var dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+      double[,] inputMatrix = dataset.ToArray(allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      if (inputMatrix.ContainsNanOrInfinity())
+        throw new NotSupportedException("Neural network ensemble regression does not support NaN or infinity values in the input dataset.");
+
+      alglib.mlpensemble mlpEnsemble = null;
+      if (nLayers == 0) {
+        alglib.mlpecreate0(allowedInputVariables.Count(), 1, ensembleSize, out mlpEnsemble);
+      } else if (nLayers == 1) {
+        alglib.mlpecreate1(allowedInputVariables.Count(), nHiddenNodes1, 1, ensembleSize, out mlpEnsemble);
+      } else if (nLayers == 2) {
+        alglib.mlpecreate2(allowedInputVariables.Count(), nHiddenNodes1, nHiddenNodes2, 1, ensembleSize, out mlpEnsemble);
+      } else throw new ArgumentException("Number of layers must be zero, one, or two.", "nLayers");
+      alglib.mlpreport rep;
+      int nRows = inputMatrix.GetLength(0);
+
+      int info;
+      alglib.mlpetraines(mlpEnsemble, inputMatrix, nRows, decay, restarts, out info, out rep);
+      if (info != 6) throw new ArgumentException("Error in calculation of neural network ensemble regression solution");
+
+      rmsError = alglib.mlpermserror(mlpEnsemble, inputMatrix, nRows);
+      avgRelError = alglib.mlpeavgrelerror(mlpEnsemble, inputMatrix, nRows);
+
+      return new NeuralNetworkEnsembleRegressionSolution(new NeuralNetworkEnsembleModel(mlpEnsemble, targetVariable, allowedInputVariables), (IRegressionProblemData)problemData.Clone());
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegressionSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegressionSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegressionSolution.cs	(revision 17041)
@@ -0,0 +1,54 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a neural network ensemble solution for a regression problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("NeuralNetworkEnsembleRegressionSolution", "Represents a neural network ensemble solution for a regression problem which can be visualized in the GUI.")]
+  [StorableType("46257A3C-9052-4390-98D1-14A6FBF51C8A")]
+  public sealed class NeuralNetworkEnsembleRegressionSolution : RegressionSolution, INeuralNetworkEnsembleRegressionSolution {
+
+    public new INeuralNetworkEnsembleModel Model {
+      get { return (INeuralNetworkEnsembleModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private NeuralNetworkEnsembleRegressionSolution(StorableConstructorFlag _) : base(_) { }
+    private NeuralNetworkEnsembleRegressionSolution(NeuralNetworkEnsembleRegressionSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NeuralNetworkEnsembleRegressionSolution(INeuralNetworkEnsembleModel nnModel, IRegressionProblemData problemData)
+      : base(nnModel, problemData) {
+      RecalculateResults();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkEnsembleRegressionSolution(this, cloner);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkModel.cs	(revision 17041)
@@ -0,0 +1,261 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a neural network model for regression and classification
+  /// </summary>
+  [StorableType("AEB9B960-FCA6-4A6D-BD5F-27BCE9CC5BEA")]
+  [Item("NeuralNetworkModel", "Represents a neural network for regression and classification.")]
+  public sealed class NeuralNetworkModel : ClassificationModel, INeuralNetworkModel {
+
+    private object mlpLocker = new object();
+    private alglib.multilayerperceptron multiLayerPerceptron;
+
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return allowedInputVariables; }
+    }
+
+    [Storable]
+    private string[] allowedInputVariables;
+    [Storable]
+    private double[] classValues;
+    [StorableConstructor]
+    private NeuralNetworkModel(StorableConstructorFlag _) : base(_) {
+      multiLayerPerceptron = new alglib.multilayerperceptron();
+    }
+    private NeuralNetworkModel(NeuralNetworkModel original, Cloner cloner)
+      : base(original, cloner) {
+      multiLayerPerceptron = new alglib.multilayerperceptron();
+      multiLayerPerceptron.innerobj.chunks = (double[,])original.multiLayerPerceptron.innerobj.chunks.Clone();
+      multiLayerPerceptron.innerobj.columnmeans = (double[])original.multiLayerPerceptron.innerobj.columnmeans.Clone();
+      multiLayerPerceptron.innerobj.columnsigmas = (double[])original.multiLayerPerceptron.innerobj.columnsigmas.Clone();
+      multiLayerPerceptron.innerobj.derror = (double[])original.multiLayerPerceptron.innerobj.derror.Clone();
+      multiLayerPerceptron.innerobj.dfdnet = (double[])original.multiLayerPerceptron.innerobj.dfdnet.Clone();
+      multiLayerPerceptron.innerobj.neurons = (double[])original.multiLayerPerceptron.innerobj.neurons.Clone();
+      multiLayerPerceptron.innerobj.nwbuf = (double[])original.multiLayerPerceptron.innerobj.nwbuf.Clone();
+      multiLayerPerceptron.innerobj.structinfo = (int[])original.multiLayerPerceptron.innerobj.structinfo.Clone();
+      multiLayerPerceptron.innerobj.weights = (double[])original.multiLayerPerceptron.innerobj.weights.Clone();
+      multiLayerPerceptron.innerobj.x = (double[])original.multiLayerPerceptron.innerobj.x.Clone();
+      multiLayerPerceptron.innerobj.y = (double[])original.multiLayerPerceptron.innerobj.y.Clone();
+      allowedInputVariables = (string[])original.allowedInputVariables.Clone();
+      if (original.classValues != null)
+        this.classValues = (double[])original.classValues.Clone();
+    }
+    public NeuralNetworkModel(alglib.multilayerperceptron multiLayerPerceptron, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues = null)
+      : base(targetVariable) {
+      this.name = ItemName;
+      this.description = ItemDescription;
+      this.multiLayerPerceptron = multiLayerPerceptron;
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+      if (classValues != null)
+        this.classValues = (double[])classValues.Clone();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkModel(this, cloner);
+    }
+
+    public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = dataset.ToArray(allowedInputVariables, rows);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] y = new double[1];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        // NOTE: mlpprocess changes data in multiLayerPerceptron and is therefore not thread-save!
+        lock (mlpLocker) {
+          alglib.mlpprocess(multiLayerPerceptron, x, ref y);
+        }
+        yield return y[0];
+      }
+    }
+
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = dataset.ToArray(allowedInputVariables, rows);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] y = new double[classValues.Length];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        // NOTE: mlpprocess changes data in multiLayerPerceptron and is therefore not thread-save!
+        lock (mlpLocker) {
+          alglib.mlpprocess(multiLayerPerceptron, x, ref y);
+        }
+        // find class for with the largest probability value
+        int maxProbClassIndex = 0;
+        double maxProb = y[0];
+        for (int i = 1; i < y.Length; i++) {
+          if (maxProb < y[i]) {
+            maxProb = y[i];
+            maxProbClassIndex = i;
+          }
+        }
+        yield return classValues[maxProbClassIndex];
+      }
+    }
+
+    public bool IsProblemDataCompatible(IRegressionProblemData problemData, out string errorMessage) {
+      return RegressionModel.IsProblemDataCompatible(this, problemData, out errorMessage);
+    }
+
+    public override bool IsProblemDataCompatible(IDataAnalysisProblemData problemData, out string errorMessage) {
+      if (problemData == null) throw new ArgumentNullException("problemData", "The provided problemData is null.");
+
+      var regressionProblemData = problemData as IRegressionProblemData;
+      if (regressionProblemData != null)
+        return IsProblemDataCompatible(regressionProblemData, out errorMessage);
+
+      var classificationProblemData = problemData as IClassificationProblemData;
+      if (classificationProblemData != null)
+        return IsProblemDataCompatible(classificationProblemData, out errorMessage);
+
+      throw new ArgumentException("The problem data is not compatible with this neural network. Instead a " + problemData.GetType().GetPrettyName() + " was provided.", "problemData");
+    }
+
+    public IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new NeuralNetworkRegressionSolution(this, new RegressionProblemData(problemData));
+    }
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new NeuralNetworkClassificationSolution(this, new ClassificationProblemData(problemData));
+    }
+
+    #region persistence
+    [Storable]
+    private double[,] MultiLayerPerceptronChunks {
+      get {
+        return multiLayerPerceptron.innerobj.chunks;
+      }
+      set {
+        multiLayerPerceptron.innerobj.chunks = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronColumnMeans {
+      get {
+        return multiLayerPerceptron.innerobj.columnmeans;
+      }
+      set {
+        multiLayerPerceptron.innerobj.columnmeans = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronColumnSigmas {
+      get {
+        return multiLayerPerceptron.innerobj.columnsigmas;
+      }
+      set {
+        multiLayerPerceptron.innerobj.columnsigmas = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronDError {
+      get {
+        return multiLayerPerceptron.innerobj.derror;
+      }
+      set {
+        multiLayerPerceptron.innerobj.derror = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronDfdnet {
+      get {
+        return multiLayerPerceptron.innerobj.dfdnet;
+      }
+      set {
+        multiLayerPerceptron.innerobj.dfdnet = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronNeurons {
+      get {
+        return multiLayerPerceptron.innerobj.neurons;
+      }
+      set {
+        multiLayerPerceptron.innerobj.neurons = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronNwbuf {
+      get {
+        return multiLayerPerceptron.innerobj.nwbuf;
+      }
+      set {
+        multiLayerPerceptron.innerobj.nwbuf = value;
+      }
+    }
+    [Storable]
+    private int[] MultiLayerPerceptronStuctinfo {
+      get {
+        return multiLayerPerceptron.innerobj.structinfo;
+      }
+      set {
+        multiLayerPerceptron.innerobj.structinfo = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronWeights {
+      get {
+        return multiLayerPerceptron.innerobj.weights;
+      }
+      set {
+        multiLayerPerceptron.innerobj.weights = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronX {
+      get {
+        return multiLayerPerceptron.innerobj.x;
+      }
+      set {
+        multiLayerPerceptron.innerobj.x = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronY {
+      get {
+        return multiLayerPerceptron.innerobj.y;
+      }
+      set {
+        multiLayerPerceptron.innerobj.y = value;
+      }
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs	(revision 17041)
@@ -0,0 +1,214 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Neural network regression data analysis algorithm.
+  /// </summary>
+  [Item("Neural Network Regression (NN)", "Neural network regression data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/neuralnetworks.php")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 130)]
+  [StorableType("F9EEA27B-43FF-4296-A4B9-CC024CD1778C")]
+  public sealed class NeuralNetworkRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+    private const string DecayParameterName = "Decay";
+    private const string HiddenLayersParameterName = "HiddenLayers";
+    private const string NodesInFirstHiddenLayerParameterName = "NodesInFirstHiddenLayer";
+    private const string NodesInSecondHiddenLayerParameterName = "NodesInSecondHiddenLayer";
+    private const string RestartsParameterName = "Restarts";
+    private const string NeuralNetworkRegressionModelResultName = "Neural network regression solution";
+
+    #region parameter properties
+    public IFixedValueParameter<DoubleValue> DecayParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[DecayParameterName]; }
+    }
+    public IConstrainedValueParameter<IntValue> HiddenLayersParameter {
+      get { return (IConstrainedValueParameter<IntValue>)Parameters[HiddenLayersParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInFirstHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInFirstHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInSecondHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInSecondHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> RestartsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[RestartsParameterName]; }
+    }
+    #endregion
+
+    #region properties
+    public double Decay {
+      get { return DecayParameter.Value.Value; }
+      set {
+        if (value < 0.001 || value > 100) throw new ArgumentException("The decay parameter should be set to a value between 0.001 and 100.", "Decay");
+        DecayParameter.Value.Value = value;
+      }
+    }
+    public int HiddenLayers {
+      get { return HiddenLayersParameter.Value.Value; }
+      set {
+        if (value < 0 || value > 2) throw new ArgumentException("The number of hidden layers should be set to 0, 1, or 2.", "HiddenLayers");
+        HiddenLayersParameter.Value = (from v in HiddenLayersParameter.ValidValues
+                                       where v.Value == value
+                                       select v)
+                                      .Single();
+      }
+    }
+    public int NodesInFirstHiddenLayer {
+      get { return NodesInFirstHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first hidden layer must be at least one.", "NodesInFirstHiddenLayer");
+        NodesInFirstHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int NodesInSecondHiddenLayer {
+      get { return NodesInSecondHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first second layer must be at least one.", "NodesInSecondHiddenLayer");
+        NodesInSecondHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int Restarts {
+      get { return RestartsParameter.Value.Value; }
+      set {
+        if (value < 0) throw new ArgumentException("The number of restarts must be positive.", "Restarts");
+        RestartsParameter.Value.Value = value;
+      }
+    }
+    #endregion
+
+
+    [StorableConstructor]
+    private NeuralNetworkRegression(StorableConstructorFlag _) : base(_) { }
+    private NeuralNetworkRegression(NeuralNetworkRegression original, Cloner cloner)
+      : base(original, cloner) {
+      RegisterEventHandlers();
+    }
+    public NeuralNetworkRegression()
+      : base() {
+      var validHiddenLayerValues = new ItemSet<IntValue>(new IntValue[] {
+        (IntValue)new IntValue(0).AsReadOnly(),
+        (IntValue)new IntValue(1).AsReadOnly(),
+        (IntValue)new IntValue(2).AsReadOnly() });
+      var selectedHiddenLayerValue = (from v in validHiddenLayerValues
+                                      where v.Value == 1
+                                      select v)
+                                     .Single();
+      Parameters.Add(new FixedValueParameter<DoubleValue>(DecayParameterName, "The decay parameter for the training phase of the neural network. This parameter determines the strengh of regularization and should be set to a value between 0.001 (weak regularization) to 100 (very strong regularization). The correct value should be determined via cross-validation.", new DoubleValue(1)));
+      Parameters.Add(new ConstrainedValueParameter<IntValue>(HiddenLayersParameterName, "The number of hidden layers for the neural network (0, 1, or 2)", validHiddenLayerValues, selectedHiddenLayerValue));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInFirstHiddenLayerParameterName, "The number of nodes in the first hidden layer. This value is not used if the number of hidden layers is zero.", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInSecondHiddenLayerParameterName, "The number of nodes in the second hidden layer. This value is not used if the number of hidden layers is zero or one.", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of restarts for learning.", new IntValue(2)));
+
+      RestartsParameter.Hidden = true;
+      NodesInSecondHiddenLayerParameter.Hidden = true;
+
+      RegisterEventHandlers();
+
+      Problem = new RegressionProblem();
+    }
+
+    private void RegisterEventHandlers() {
+      HiddenLayersParameter.Value.ValueChanged += HiddenLayersParameterValueValueChanged;
+      HiddenLayersParameter.ValueChanged += HiddenLayersParameterValueChanged;
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      RegisterEventHandlers();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkRegression(this, cloner);
+    }
+
+    private void HiddenLayersParameterValueChanged(object source, EventArgs e) {
+      HiddenLayersParameter.Value.ValueChanged += HiddenLayersParameterValueValueChanged;
+      HiddenLayersParameterValueValueChanged(this, EventArgs.Empty);
+    }
+
+    private void HiddenLayersParameterValueValueChanged(object source, EventArgs e) {
+      if (HiddenLayers == 0) {
+        NodesInFirstHiddenLayerParameter.Hidden = true;
+        NodesInSecondHiddenLayerParameter.Hidden = true;
+      } else if (HiddenLayers == 1) {
+        NodesInFirstHiddenLayerParameter.Hidden = false;
+        NodesInSecondHiddenLayerParameter.Hidden = true;
+      } else {
+        NodesInFirstHiddenLayerParameter.Hidden = false;
+        NodesInSecondHiddenLayerParameter.Hidden = false;
+      }
+    }
+
+
+    #region neural network
+    protected override void Run(CancellationToken cancellationToken) {
+      double rmsError, avgRelError;
+      var solution = CreateNeuralNetworkRegressionSolution(Problem.ProblemData, HiddenLayers, NodesInFirstHiddenLayer, NodesInSecondHiddenLayer, Decay, Restarts, out rmsError, out avgRelError);
+      Results.Add(new Result(NeuralNetworkRegressionModelResultName, "The neural network regression solution.", solution));
+      Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the neural network regression solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Average relative error", "The average of relative errors of the neural network regression solution on the training set.", new PercentValue(avgRelError)));
+    }
+
+    public static IRegressionSolution CreateNeuralNetworkRegressionSolution(IRegressionProblemData problemData, int nLayers, int nHiddenNodes1, int nHiddenNodes2, double decay, int restarts,
+      out double rmsError, out double avgRelError) {
+      var dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+      double[,] inputMatrix = dataset.ToArray(allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      if (inputMatrix.ContainsNanOrInfinity())
+        throw new NotSupportedException("Neural network regression does not support NaN or infinity values in the input dataset.");
+
+      alglib.multilayerperceptron multiLayerPerceptron = null;
+      if (nLayers == 0) {
+        alglib.mlpcreate0(allowedInputVariables.Count(), 1, out multiLayerPerceptron);
+      } else if (nLayers == 1) {
+        alglib.mlpcreate1(allowedInputVariables.Count(), nHiddenNodes1, 1, out multiLayerPerceptron);
+      } else if (nLayers == 2) {
+        alglib.mlpcreate2(allowedInputVariables.Count(), nHiddenNodes1, nHiddenNodes2, 1, out multiLayerPerceptron);
+      } else throw new ArgumentException("Number of layers must be zero, one, or two.", "nLayers");
+      alglib.mlpreport rep;
+      int nRows = inputMatrix.GetLength(0);
+
+      int info;
+      // using mlptrainlm instead of mlptraines or mlptrainbfgs because only one parameter is necessary
+      alglib.mlptrainlm(multiLayerPerceptron, inputMatrix, nRows, decay, restarts, out info, out rep);
+      if (info != 2) throw new ArgumentException("Error in calculation of neural network regression solution");
+
+      rmsError = alglib.mlprmserror(multiLayerPerceptron, inputMatrix, nRows);
+      avgRelError = alglib.mlpavgrelerror(multiLayerPerceptron, inputMatrix, nRows);
+
+      return new NeuralNetworkRegressionSolution(new NeuralNetworkModel(multiLayerPerceptron, targetVariable, allowedInputVariables), (IRegressionProblemData)problemData.Clone());
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegressionSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegressionSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegressionSolution.cs	(revision 17041)
@@ -0,0 +1,53 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a neural network solution for a regression problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("NeuralNetworkRegressionSolution", "Represents a neural network solution for a regression problem which can be visualized in the GUI.")]
+  [StorableType("E1E1B29A-B87A-4BCA-9A2C-83DA8AD7B9D9")]
+  public sealed class NeuralNetworkRegressionSolution : RegressionSolution, INeuralNetworkRegressionSolution {
+
+    public new INeuralNetworkModel Model {
+      get { return (INeuralNetworkModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private NeuralNetworkRegressionSolution(StorableConstructorFlag _) : base(_) { }
+    private NeuralNetworkRegressionSolution(NeuralNetworkRegressionSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NeuralNetworkRegressionSolution(INeuralNetworkModel nnModel, IRegressionProblemData problemData)
+      : base(nnModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkRegressionSolution(this, cloner);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NonlinearRegression/NonlinearRegression.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NonlinearRegression/NonlinearRegression.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/NonlinearRegression/NonlinearRegression.cs	(revision 17041)
@@ -0,0 +1,292 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Analysis;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
+using HeuristicLab.Random;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Nonlinear regression data analysis algorithm.
+  /// </summary>
+  [Item("Nonlinear Regression (NLR)", "Nonlinear regression (curve fitting) data analysis algorithm (wrapper for ALGLIB).")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 120)]
+  [StorableType("06E970EA-D366-4F46-BDC5-7156B5787BEF")]
+  public sealed class NonlinearRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+    private const string RegressionSolutionResultName = "Regression solution";
+    private const string ModelStructureParameterName = "Model structure";
+    private const string IterationsParameterName = "Iterations";
+    private const string RestartsParameterName = "Restarts";
+    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+    private const string SeedParameterName = "Seed";
+    private const string InitParamsRandomlyParameterName = "InitializeParametersRandomly";
+    private const string ApplyLinearScalingParameterName = "Apply linear scaling";
+
+    public IFixedValueParameter<StringValue> ModelStructureParameter {
+      get { return (IFixedValueParameter<StringValue>)Parameters[ModelStructureParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> IterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[IterationsParameterName]; }
+    }
+
+    public IFixedValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+
+    public IFixedValueParameter<IntValue> SeedParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+
+    public IFixedValueParameter<IntValue> RestartsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[RestartsParameterName]; }
+    }
+
+    public IFixedValueParameter<BoolValue> InitParametersRandomlyParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[InitParamsRandomlyParameterName]; }
+    }
+
+    public IFixedValueParameter<BoolValue> ApplyLinearScalingParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[ApplyLinearScalingParameterName]; }
+    }
+
+    public string ModelStructure {
+      get { return ModelStructureParameter.Value.Value; }
+      set { ModelStructureParameter.Value.Value = value; }
+    }
+
+    public int Iterations {
+      get { return IterationsParameter.Value.Value; }
+      set { IterationsParameter.Value.Value = value; }
+    }
+
+    public int Restarts {
+      get { return RestartsParameter.Value.Value; }
+      set { RestartsParameter.Value.Value = value; }
+    }
+
+    public int Seed {
+      get { return SeedParameter.Value.Value; }
+      set { SeedParameter.Value.Value = value; }
+    }
+
+    public bool SetSeedRandomly {
+      get { return SetSeedRandomlyParameter.Value.Value; }
+      set { SetSeedRandomlyParameter.Value.Value = value; }
+    }
+
+    public bool InitializeParametersRandomly {
+      get { return InitParametersRandomlyParameter.Value.Value; }
+      set { InitParametersRandomlyParameter.Value.Value = value; }
+    }
+
+    public bool ApplyLinearScaling {
+      get { return ApplyLinearScalingParameter.Value.Value; }
+      set { ApplyLinearScalingParameter.Value.Value = value; }
+    }
+
+    [StorableConstructor]
+    private NonlinearRegression(StorableConstructorFlag _) : base(_) { }
+    private NonlinearRegression(NonlinearRegression original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NonlinearRegression()
+      : base() {
+      Problem = new RegressionProblem();
+      Parameters.Add(new FixedValueParameter<StringValue>(ModelStructureParameterName, "The function for which the parameters must be fit (only numeric constants are tuned).", new StringValue("1.0 * x*x + 0.0")));
+      Parameters.Add(new FixedValueParameter<IntValue>(IterationsParameterName, "The maximum number of iterations for constants optimization.", new IntValue(200)));
+      Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of independent random restarts (>0)", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The PRNG seed value.", new IntValue()));
+      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "Switch to determine if the random number seed should be initialized randomly.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(InitParamsRandomlyParameterName, "Switch to determine if the real-valued model parameters should be initialized randomly in each restart.", new BoolValue(false)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(ApplyLinearScalingParameterName, "Switch to determine if linear scaling terms should be added to the model", new BoolValue(true)));
+
+      SetParameterHiddenState();
+
+      InitParametersRandomlyParameter.Value.ValueChanged += (sender, args) => {
+        SetParameterHiddenState();
+      };
+    }
+
+    private void SetParameterHiddenState() {
+      var hide = !InitializeParametersRandomly;
+      RestartsParameter.Hidden = hide;
+      SeedParameter.Hidden = hide;
+      SetSeedRandomlyParameter.Hidden = hide;
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
+      if (!Parameters.ContainsKey(RestartsParameterName))
+        Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of independent random restarts", new IntValue(1)));
+      if (!Parameters.ContainsKey(SeedParameterName))
+        Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The PRNG seed value.", new IntValue()));
+      if (!Parameters.ContainsKey(SetSeedRandomlyParameterName))
+        Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "Switch to determine if the random number seed should be initialized randomly.", new BoolValue(true)));
+      if (!Parameters.ContainsKey(InitParamsRandomlyParameterName))
+        Parameters.Add(new FixedValueParameter<BoolValue>(InitParamsRandomlyParameterName, "Switch to determine if the numeric parameters of the model should be initialized randomly.", new BoolValue(false)));
+      if (!Parameters.ContainsKey(ApplyLinearScalingParameterName))
+        Parameters.Add(new FixedValueParameter<BoolValue>(ApplyLinearScalingParameterName, "Switch to determine if linear scaling terms should be added to the model", new BoolValue(true)));
+
+
+      SetParameterHiddenState();
+      InitParametersRandomlyParameter.Value.ValueChanged += (sender, args) => {
+        SetParameterHiddenState();
+      };
+      #endregion
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NonlinearRegression(this, cloner);
+    }
+
+    #region nonlinear regression
+    protected override void Run(CancellationToken cancellationToken) {
+      IRegressionSolution bestSolution = null;
+      if (InitializeParametersRandomly) {
+        var qualityTable = new DataTable("RMSE table");
+        qualityTable.VisualProperties.YAxisLogScale = true;
+        var trainRMSERow = new DataRow("RMSE (train)");
+        trainRMSERow.VisualProperties.ChartType = DataRowVisualProperties.DataRowChartType.Points;
+        var testRMSERow = new DataRow("RMSE test");
+        testRMSERow.VisualProperties.ChartType = DataRowVisualProperties.DataRowChartType.Points;
+
+        qualityTable.Rows.Add(trainRMSERow);
+        qualityTable.Rows.Add(testRMSERow);
+        Results.Add(new Result(qualityTable.Name, qualityTable.Name + " for all restarts", qualityTable));
+        if (SetSeedRandomly) Seed = RandomSeedGenerator.GetSeed();
+        var rand = new MersenneTwister((uint)Seed);
+        bestSolution = CreateRegressionSolution(Problem.ProblemData, ModelStructure, Iterations, ApplyLinearScaling, rand);
+        trainRMSERow.Values.Add(bestSolution.TrainingRootMeanSquaredError);
+        testRMSERow.Values.Add(bestSolution.TestRootMeanSquaredError);
+        for (int r = 0; r < Restarts; r++) {
+          var solution = CreateRegressionSolution(Problem.ProblemData, ModelStructure, Iterations, ApplyLinearScaling, rand);
+          trainRMSERow.Values.Add(solution.TrainingRootMeanSquaredError);
+          testRMSERow.Values.Add(solution.TestRootMeanSquaredError);
+          if (solution.TrainingRootMeanSquaredError < bestSolution.TrainingRootMeanSquaredError) {
+            bestSolution = solution;
+          }
+        }
+      } else {
+        bestSolution = CreateRegressionSolution(Problem.ProblemData, ModelStructure, Iterations, ApplyLinearScaling);
+      }
+
+      Results.Add(new Result(RegressionSolutionResultName, "The nonlinear regression solution.", bestSolution));
+      Results.Add(new Result("Root mean square error (train)", "The root of the mean of squared errors of the regression solution on the training set.", new DoubleValue(bestSolution.TrainingRootMeanSquaredError)));
+      Results.Add(new Result("Root mean square error (test)", "The root of the mean of squared errors of the regression solution on the test set.", new DoubleValue(bestSolution.TestRootMeanSquaredError)));
+
+    }
+
+    /// <summary>
+    /// Fits a model to the data by optimizing the numeric constants.
+    /// Model is specified as infix expression containing variable names and numbers. 
+    /// The starting point for the numeric constants is initialized randomly if a random number generator is specified (~N(0,1)). Otherwise the user specified constants are
+    /// used as a starting point. 
+    /// </summary>-
+    /// <param name="problemData">Training and test data</param>
+    /// <param name="modelStructure">The function as infix expression</param>
+    /// <param name="maxIterations">Number of constant optimization iterations (using Levenberg-Marquardt algorithm)</param>
+    /// <param name="random">Optional random number generator for random initialization of numeric constants.</param>
+    /// <returns></returns>
+    public static ISymbolicRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData, string modelStructure, int maxIterations, bool applyLinearScaling, IRandom rand = null) {
+      var parser = new InfixExpressionParser();
+      var tree = parser.Parse(modelStructure);
+      // parser handles double and string variables equally by creating a VariableTreeNode
+      // post-process to replace VariableTreeNodes by FactorVariableTreeNodes for all string variables
+      var factorSymbol = new FactorVariable();
+      factorSymbol.VariableNames =
+        problemData.AllowedInputVariables.Where(name => problemData.Dataset.VariableHasType<string>(name));
+      factorSymbol.AllVariableNames = factorSymbol.VariableNames;
+      factorSymbol.VariableValues =
+        factorSymbol.VariableNames.Select(name =>
+        new KeyValuePair<string, Dictionary<string, int>>(name,
+        problemData.Dataset.GetReadOnlyStringValues(name).Distinct()
+        .Select((n, i) => Tuple.Create(n, i))
+        .ToDictionary(tup => tup.Item1, tup => tup.Item2)));
+
+      foreach (var parent in tree.IterateNodesPrefix().ToArray()) {
+        for (int i = 0; i < parent.SubtreeCount; i++) {
+          var varChild = parent.GetSubtree(i) as VariableTreeNode;
+          var factorVarChild = parent.GetSubtree(i) as FactorVariableTreeNode;
+          if (varChild != null && factorSymbol.VariableNames.Contains(varChild.VariableName)) {
+            parent.RemoveSubtree(i);
+            var factorTreeNode = (FactorVariableTreeNode)factorSymbol.CreateTreeNode();
+            factorTreeNode.VariableName = varChild.VariableName;
+            factorTreeNode.Weights =
+              factorTreeNode.Symbol.GetVariableValues(factorTreeNode.VariableName).Select(_ => 1.0).ToArray();
+            // weight = 1.0 for each value
+            parent.InsertSubtree(i, factorTreeNode);
+          } else if (factorVarChild != null && factorSymbol.VariableNames.Contains(factorVarChild.VariableName)) {
+            if (factorSymbol.GetVariableValues(factorVarChild.VariableName).Count() != factorVarChild.Weights.Length)
+              throw new ArgumentException(
+                string.Format("Factor variable {0} needs exactly {1} weights",
+                factorVarChild.VariableName,
+                factorSymbol.GetVariableValues(factorVarChild.VariableName).Count()));
+            parent.RemoveSubtree(i);
+            var factorTreeNode = (FactorVariableTreeNode)factorSymbol.CreateTreeNode();
+            factorTreeNode.VariableName = factorVarChild.VariableName;
+            factorTreeNode.Weights = factorVarChild.Weights;
+            parent.InsertSubtree(i, factorTreeNode);
+          }
+        }
+      }
+
+      if (!SymbolicRegressionConstantOptimizationEvaluator.CanOptimizeConstants(tree)) throw new ArgumentException("The optimizer does not support the specified model structure.");
+
+      // initialize constants randomly
+      if (rand != null) {
+        foreach (var node in tree.IterateNodesPrefix().OfType<ConstantTreeNode>()) {
+          double f = Math.Exp(NormalDistributedRandom.NextDouble(rand, 0, 1));
+          double s = rand.NextDouble() < 0.5 ? -1 : 1;
+          node.Value = s * node.Value * f;
+        }
+      }
+      var interpreter = new SymbolicDataAnalysisExpressionTreeLinearInterpreter();
+
+      SymbolicRegressionConstantOptimizationEvaluator.OptimizeConstants(interpreter, tree, problemData, problemData.TrainingIndices,
+        applyLinearScaling: applyLinearScaling, maxIterations: maxIterations,
+        updateVariableWeights: false, updateConstantsInTree: true);
+
+      var model = new SymbolicRegressionModel(problemData.TargetVariable, tree, (ISymbolicDataAnalysisExpressionTreeInterpreter)interpreter.Clone());
+      if (applyLinearScaling)
+        model.Scale(problemData);
+
+      SymbolicRegressionSolution solution = new SymbolicRegressionSolution(model, (IRegressionProblemData)problemData.Clone());
+      solution.Model.Name = "Regression Model";
+      solution.Name = "Regression Solution";
+      return solution;
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame	(revision 17041)
@@ -0,0 +1,56 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.PluginInfrastructure;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Plugin class for HeuristicLab.Algorithms.DataAnalysis plugin.
+  /// </summary>
+  [Plugin("HeuristicLab.Algorithms.DataAnalysis", "Provides wrappers for data analysis algorithms implemented in external libraries (linear regression, linear discriminant analysis, k-means clustering, support vector classification and regression)", "3.4.11.$WCREV$")]
+  [PluginFile("HeuristicLab.Algorithms.DataAnalysis-3.4.dll", PluginFileType.Assembly)]
+  [PluginDependency("HeuristicLab.ALGLIB", "3.7.0")]
+  [PluginDependency("HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm", "3.3")] // for GBM
+  [PluginDependency("HeuristicLab.Algorithms.GradientDescent", "3.3")]
+  [PluginDependency("HeuristicLab.Analysis", "3.3")]
+  [PluginDependency("HeuristicLab.Collections", "3.3")]
+  [PluginDependency("HeuristicLab.Common", "3.3")]
+  [PluginDependency("HeuristicLab.Common.Resources", "3.3")]
+  [PluginDependency("HeuristicLab.Core", "3.3")]
+  [PluginDependency("HeuristicLab.Data", "3.3")]
+  [PluginDependency("HeuristicLab.Encodings.RealVectorEncoding", "3.3")]
+  [PluginDependency("HeuristicLab.Encodings.SymbolicExpressionTreeEncoding", "3.4")]
+  [PluginDependency("HeuristicLab.Operators", "3.3")]
+  [PluginDependency("HeuristicLab.Optimization", "3.3")]
+  [PluginDependency("HeuristicLab.Parameters", "3.3")]
+  [PluginDependency("HeuristicLab.Attic", "1.0")]
+  [PluginDependency("HeuristicLab.Problems.DataAnalysis", "3.4")]
+  [PluginDependency("HeuristicLab.Problems.DataAnalysis.Symbolic", "3.4")]
+  [PluginDependency("HeuristicLab.Problems.DataAnalysis.Symbolic.Classification", "3.4")]
+  [PluginDependency("HeuristicLab.Problems.DataAnalysis.Symbolic.Regression", "3.4")]
+  [PluginDependency("HeuristicLab.Problems.DataAnalysis.Symbolic.TimeSeriesPrognosis", "3.4")]
+  [PluginDependency("HeuristicLab.Problems.Instances", "3.3")]
+  [PluginDependency("HeuristicLab.LibSVM", "3.12")]
+  [PluginDependency("HeuristicLab.Random", "3.3")]
+  [PluginDependency("HeuristicLab.Selection", "3.3")]
+  public class HeuristicLabAlgorithmsDataAnalysisPlugin : PluginBase {
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Properties/AssemblyInfo.cs.frame
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Properties/AssemblyInfo.cs.frame	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/Properties/AssemblyInfo.cs.frame	(revision 17041)
@@ -0,0 +1,55 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Reflection;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("HeuristicLab.Algorithms.DataAnalysis")]
+[assembly: AssemblyDescription("Provides wrappers for data analysis algorithms implemented in external libraries (linear regression, linear discriminant analysis, k-means clustering, support vector classification and regression)")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("HeuristicLab")]
+[assembly: AssemblyCopyright("(c) 2002-2018 HEAL")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("3E422463-0CE0-400D-9544-5C11D5818769")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Revision and Build Numbers 
+// by using the '*' as shown below:
+[assembly: AssemblyVersion("3.4.0.0")]
+[assembly: AssemblyFileVersion("3.4.11.$WCREV$")]
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassification.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassification.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassification.cs	(revision 17041)
@@ -0,0 +1,166 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Random forest classification data analysis algorithm.
+  /// </summary>
+  [Item("Random Forest Classification (RF)", "Random forest classification data analysis algorithm (wrapper for ALGLIB).")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 120)]
+  [StorableType("73070CC7-E85E-4851-9F26-C537AE1CC1C0")]
+  public sealed class RandomForestClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+    private const string RandomForestClassificationModelResultName = "Random forest classification solution";
+    private const string NumberOfTreesParameterName = "Number of trees";
+    private const string RParameterName = "R";
+    private const string MParameterName = "M";
+    private const string SeedParameterName = "Seed";
+    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+    private const string CreateSolutionParameterName = "CreateSolution";
+
+    #region parameter properties
+    public IFixedValueParameter<IntValue> NumberOfTreesParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NumberOfTreesParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> RParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[RParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> MParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[MParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> SeedParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> CreateSolutionParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
+    }
+    #endregion
+    #region properties
+    public int NumberOfTrees {
+      get { return NumberOfTreesParameter.Value.Value; }
+      set { NumberOfTreesParameter.Value.Value = value; }
+    }
+    public double R {
+      get { return RParameter.Value.Value; }
+      set { RParameter.Value.Value = value; }
+    }
+    public double M {
+      get { return MParameter.Value.Value; }
+      set { MParameter.Value.Value = value; }
+    }
+    public int Seed {
+      get { return SeedParameter.Value.Value; }
+      set { SeedParameter.Value.Value = value; }
+    }
+    public bool SetSeedRandomly {
+      get { return SetSeedRandomlyParameter.Value.Value; }
+      set { SetSeedRandomlyParameter.Value.Value = value; }
+    }
+    public bool CreateSolution {
+      get { return CreateSolutionParameter.Value.Value; }
+      set { CreateSolutionParameter.Value.Value = value; }
+    }
+    #endregion
+
+    [StorableConstructor]
+    private RandomForestClassification(StorableConstructorFlag _) : base(_) { }
+    private RandomForestClassification(RandomForestClassification original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public RandomForestClassification()
+      : base() {
+      Parameters.Add(new FixedValueParameter<IntValue>(NumberOfTreesParameterName, "The number of trees in the forest. Should be between 50 and 100", new IntValue(50)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(RParameterName, "The ratio of the training set that will be used in the construction of individual trees (0<r<=1). Should be adjusted depending on the noise level in the dataset in the range from 0.66 (low noise) to 0.05 (high noise). This parameter should be adjusted to achieve good generalization error.", new DoubleValue(0.3)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName, "The ratio of features that will be used in the construction of individual trees (0<m<=1)", new DoubleValue(0.5)));
+      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+      Parameters[CreateSolutionParameterName].Hidden = true;
+
+      Problem = new ClassificationProblem();
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
+      if (!Parameters.ContainsKey(MParameterName))
+        Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName, "The ratio of features that will be used in the construction of individual trees (0<m<=1)", new DoubleValue(0.5)));
+      if (!Parameters.ContainsKey(SeedParameterName))
+        Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+      if (!Parameters.ContainsKey((SetSeedRandomlyParameterName)))
+        Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+      if (!Parameters.ContainsKey(CreateSolutionParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+        Parameters[CreateSolutionParameterName].Hidden = true;
+      }
+      #endregion
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RandomForestClassification(this, cloner);
+    }
+
+    #region random forest
+    protected override void Run(CancellationToken cancellationToken) {
+      double rmsError, relClassificationError, outOfBagRmsError, outOfBagRelClassificationError;
+      if (SetSeedRandomly) Seed = Random.RandomSeedGenerator.GetSeed();
+
+      var model = CreateRandomForestClassificationModel(Problem.ProblemData, NumberOfTrees, R, M, Seed, out rmsError, out relClassificationError, out outOfBagRmsError, out outOfBagRelClassificationError);
+      Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the random forest regression solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Relative classification error", "Relative classification error of the random forest regression solution on the training set.", new PercentValue(relClassificationError)));
+      Results.Add(new Result("Root mean square error (out-of-bag)", "The out-of-bag root of the mean of squared errors of the random forest regression solution.", new DoubleValue(outOfBagRmsError)));
+      Results.Add(new Result("Relative classification error (out-of-bag)", "The out-of-bag relative classification error  of the random forest regression solution.", new PercentValue(outOfBagRelClassificationError)));
+
+      if (CreateSolution) {
+        var solution = new RandomForestClassificationSolution(model, (IClassificationProblemData)Problem.ProblemData.Clone());
+        Results.Add(new Result(RandomForestClassificationModelResultName, "The random forest classification solution.", solution));
+      }
+    }
+
+    // keep for compatibility with old API
+    public static RandomForestClassificationSolution CreateRandomForestClassificationSolution(IClassificationProblemData problemData, int nTrees, double r, double m, int seed,
+      out double rmsError, out double relClassificationError, out double outOfBagRmsError, out double outOfBagRelClassificationError) {
+      var model = CreateRandomForestClassificationModel(problemData, nTrees, r, m, seed,
+        out rmsError, out relClassificationError, out outOfBagRmsError, out outOfBagRelClassificationError);
+      return new RandomForestClassificationSolution(model, (IClassificationProblemData)problemData.Clone());
+    }
+
+    public static RandomForestModel CreateRandomForestClassificationModel(IClassificationProblemData problemData, int nTrees, double r, double m, int seed,
+      out double rmsError, out double relClassificationError, out double outOfBagRmsError, out double outOfBagRelClassificationError) {
+      return RandomForestModel.CreateClassificationModel(problemData, nTrees, r, m, seed,
+       rmsError: out rmsError, relClassificationError: out relClassificationError, outOfBagRmsError: out outOfBagRmsError, outOfBagRelClassificationError: out outOfBagRelClassificationError);
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassificationSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassificationSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassificationSolution.cs	(revision 17041)
@@ -0,0 +1,57 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a random forest solution for a classification problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("RandomForestClassificationSolution", "Represents a random forest solution for a classification problem which can be visualized in the GUI.")]
+  [StorableType("11D108AD-931C-4504-BA87-0D5CEB2C95A9")]
+  public sealed class RandomForestClassificationSolution : ClassificationSolution, IRandomForestClassificationSolution {
+
+    public new IRandomForestModel Model {
+      get { return (IRandomForestModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    public int NumberOfTrees {
+      get { return Model.NumberOfTrees; }
+    }
+
+    [StorableConstructor]
+    private RandomForestClassificationSolution(StorableConstructorFlag _) : base(_) { }
+    private RandomForestClassificationSolution(RandomForestClassificationSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public RandomForestClassificationSolution(IRandomForestModel randomForestModel, IClassificationProblemData problemData)
+      : base(randomForestModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RandomForestClassificationSolution(this, cloner);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs	(revision 17041)
@@ -0,0 +1,460 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a random forest model for regression and classification
+  /// </summary>
+  [StorableType("A4F688CD-1F42-4103-8449-7DE52AEF6C69")]
+  [Item("RandomForestModel", "Represents a random forest for regression and classification.")]
+  public sealed class RandomForestModel : ClassificationModel, IRandomForestModel {
+    // not persisted
+    private alglib.decisionforest randomForest;
+    private alglib.decisionforest RandomForest {
+      get {
+        // recalculate lazily
+        if (randomForest.innerobj.trees == null || randomForest.innerobj.trees.Length == 0) RecalculateModel();
+        return randomForest;
+      }
+    }
+
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return originalTrainingData.AllowedInputVariables; }
+    }
+
+    public int NumberOfTrees {
+      get { return nTrees; }
+    }
+
+    // instead of storing the data of the model itself
+    // we instead only store data necessary to recalculate the same model lazily on demand
+    [Storable]
+    private int seed;
+    [Storable]
+    private IDataAnalysisProblemData originalTrainingData;
+    [Storable]
+    private double[] classValues;
+    [Storable]
+    private int nTrees;
+    [Storable]
+    private double r;
+    [Storable]
+    private double m;
+
+    [StorableConstructor]
+    private RandomForestModel(StorableConstructorFlag _) : base(_) {
+      // for backwards compatibility (loading old solutions)
+      randomForest = new alglib.decisionforest();
+    }
+    private RandomForestModel(RandomForestModel original, Cloner cloner)
+      : base(original, cloner) {
+      randomForest = new alglib.decisionforest();
+      randomForest.innerobj.bufsize = original.randomForest.innerobj.bufsize;
+      randomForest.innerobj.nclasses = original.randomForest.innerobj.nclasses;
+      randomForest.innerobj.ntrees = original.randomForest.innerobj.ntrees;
+      randomForest.innerobj.nvars = original.randomForest.innerobj.nvars;
+      // we assume that the trees array (double[]) is immutable in alglib
+      randomForest.innerobj.trees = original.randomForest.innerobj.trees;
+
+      // allowedInputVariables is immutable so we don't need to clone
+      allowedInputVariables = original.allowedInputVariables;
+
+      // clone data which is necessary to rebuild the model
+      this.seed = original.seed;
+      this.originalTrainingData = cloner.Clone(original.originalTrainingData);
+      // classvalues is immutable so we don't need to clone
+      this.classValues = original.classValues;
+      this.nTrees = original.nTrees;
+      this.r = original.r;
+      this.m = original.m;
+    }
+
+    // random forest models can only be created through the static factory methods CreateRegressionModel and CreateClassificationModel
+    private RandomForestModel(string targetVariable, alglib.decisionforest randomForest,
+      int seed, IDataAnalysisProblemData originalTrainingData,
+      int nTrees, double r, double m, double[] classValues = null)
+      : base(targetVariable) {
+      this.name = ItemName;
+      this.description = ItemDescription;
+      // the model itself
+      this.randomForest = randomForest;
+      // data which is necessary for recalculation of the model
+      this.seed = seed;
+      this.originalTrainingData = (IDataAnalysisProblemData)originalTrainingData.Clone();
+      this.classValues = classValues;
+      this.nTrees = nTrees;
+      this.r = r;
+      this.m = m;
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RandomForestModel(this, cloner);
+    }
+
+    private void RecalculateModel() {
+      double rmsError, oobRmsError, relClassError, oobRelClassError;
+      var regressionProblemData = originalTrainingData as IRegressionProblemData;
+      var classificationProblemData = originalTrainingData as IClassificationProblemData;
+      if (regressionProblemData != null) {
+        var model = CreateRegressionModel(regressionProblemData,
+                                              nTrees, r, m, seed, out rmsError, out oobRmsError,
+                                              out relClassError, out oobRelClassError);
+        randomForest = model.randomForest;
+      } else if (classificationProblemData != null) {
+        var model = CreateClassificationModel(classificationProblemData,
+                                              nTrees, r, m, seed, out rmsError, out oobRmsError,
+                                              out relClassError, out oobRelClassError);
+        randomForest = model.randomForest;
+      }
+    }
+
+    public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = dataset.ToArray(AllowedInputVariables, rows);
+      AssertInputMatrix(inputData);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] y = new double[1];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        alglib.dfprocess(RandomForest, x, ref y);
+        yield return y[0];
+      }
+    }
+
+    public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = dataset.ToArray(AllowedInputVariables, rows);
+      AssertInputMatrix(inputData);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] ys = new double[this.RandomForest.innerobj.ntrees];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        alglib.dforest.dfprocessraw(RandomForest.innerobj, x, ref ys);
+        yield return ys.VariancePop();
+      }
+    }
+
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = dataset.ToArray(AllowedInputVariables, rows);
+      AssertInputMatrix(inputData);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] y = new double[RandomForest.innerobj.nclasses];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        alglib.dfprocess(randomForest, x, ref y);
+        // find class for with the largest probability value
+        int maxProbClassIndex = 0;
+        double maxProb = y[0];
+        for (int i = 1; i < y.Length; i++) {
+          if (maxProb < y[i]) {
+            maxProb = y[i];
+            maxProbClassIndex = i;
+          }
+        }
+        yield return classValues[maxProbClassIndex];
+      }
+    }
+
+    public ISymbolicExpressionTree ExtractTree(int treeIdx) {
+      var rf = RandomForest;
+      // hoping that the internal representation of alglib is stable
+
+      // TREE FORMAT
+      // W[Offs]      -   size of sub-array (for the tree)
+      //     node info:
+      // W[K+0]       -   variable number        (-1 for leaf mode)
+      // W[K+1]       -   threshold              (class/value for leaf node)
+      // W[K+2]       -   ">=" branch index      (absent for leaf node)
+
+      // skip irrelevant trees
+      int offset = 0;
+      for (int i = 0; i < treeIdx - 1; i++) {
+        offset = offset + (int)Math.Round(rf.innerobj.trees[offset]);
+      }
+
+      var constSy = new Constant();
+      var varCondSy = new VariableCondition() { IgnoreSlope = true };
+
+      var node = CreateRegressionTreeRec(rf.innerobj.trees, offset, offset + 1, constSy, varCondSy);
+
+      var startNode = new StartSymbol().CreateTreeNode();
+      startNode.AddSubtree(node);
+      var root = new ProgramRootSymbol().CreateTreeNode();
+      root.AddSubtree(startNode);
+      return new SymbolicExpressionTree(root);
+    }
+
+    private ISymbolicExpressionTreeNode CreateRegressionTreeRec(double[] trees, int offset, int k, Constant constSy, VariableCondition varCondSy) {
+
+      // alglib source for evaluation of one tree (dfprocessinternal)
+      // offs = 0
+      //
+      // Set pointer to the root
+      //
+      // k = offs + 1;
+      // 
+      // //
+      // // Navigate through the tree
+      // //
+      // while (true) {
+      //   if ((double)(df.trees[k]) == (double)(-1)) {
+      //     if (df.nclasses == 1) {
+      //       y[0] = y[0] + df.trees[k + 1];
+      //     } else {
+      //       idx = (int)Math.Round(df.trees[k + 1]);
+      //       y[idx] = y[idx] + 1;
+      //     }
+      //     break;
+      //   }
+      //   if ((double)(x[(int)Math.Round(df.trees[k])]) < (double)(df.trees[k + 1])) {
+      //     k = k + innernodewidth;
+      //   } else {
+      //     k = offs + (int)Math.Round(df.trees[k + 2]);
+      //   }
+      // }
+
+      if ((double)(trees[k]) == (double)(-1)) {
+        var constNode = (ConstantTreeNode)constSy.CreateTreeNode();
+        constNode.Value = trees[k + 1];
+        return constNode;
+      } else {
+        var condNode = (VariableConditionTreeNode)varCondSy.CreateTreeNode();
+        condNode.VariableName = AllowedInputVariables[(int)Math.Round(trees[k])];
+        condNode.Threshold = trees[k + 1];
+        condNode.Slope = double.PositiveInfinity;
+
+        var left = CreateRegressionTreeRec(trees, offset, k + 3, constSy, varCondSy);
+        var right = CreateRegressionTreeRec(trees, offset, offset + (int)Math.Round(trees[k + 2]), constSy, varCondSy);
+
+        condNode.AddSubtree(left); // not 100% correct because interpreter uses: if(x <= thres) left() else right() and RF uses if(x < thres) left() else right() (see above)
+        condNode.AddSubtree(right);
+        return condNode;
+      }
+    }
+
+
+    public IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RandomForestRegressionSolution(this, new RegressionProblemData(problemData));
+    }
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new RandomForestClassificationSolution(this, new ClassificationProblemData(problemData));
+    }
+
+    public bool IsProblemDataCompatible(IRegressionProblemData problemData, out string errorMessage) {
+      return RegressionModel.IsProblemDataCompatible(this, problemData, out errorMessage);
+    }
+
+    public override bool IsProblemDataCompatible(IDataAnalysisProblemData problemData, out string errorMessage) {
+      if (problemData == null) throw new ArgumentNullException("problemData", "The provided problemData is null.");
+
+      var regressionProblemData = problemData as IRegressionProblemData;
+      if (regressionProblemData != null)
+        return IsProblemDataCompatible(regressionProblemData, out errorMessage);
+
+      var classificationProblemData = problemData as IClassificationProblemData;
+      if (classificationProblemData != null)
+        return IsProblemDataCompatible(classificationProblemData, out errorMessage);
+
+      throw new ArgumentException("The problem data is not compatible with this random forest. Instead a " + problemData.GetType().GetPrettyName() + " was provided.", "problemData");
+    }
+
+    public static RandomForestModel CreateRegressionModel(IRegressionProblemData problemData, int nTrees, double r, double m, int seed,
+      out double rmsError, out double outOfBagRmsError, out double avgRelError, out double outOfBagAvgRelError) {
+      return CreateRegressionModel(problemData, problemData.TrainingIndices, nTrees, r, m, seed,
+       rmsError: out rmsError, outOfBagRmsError: out outOfBagRmsError, avgRelError: out avgRelError, outOfBagAvgRelError: out outOfBagAvgRelError);
+    }
+
+    public static RandomForestModel CreateRegressionModel(IRegressionProblemData problemData, IEnumerable<int> trainingIndices, int nTrees, double r, double m, int seed,
+      out double rmsError, out double outOfBagRmsError, out double avgRelError, out double outOfBagAvgRelError) {
+      var variables = problemData.AllowedInputVariables.Concat(new string[] { problemData.TargetVariable });
+      double[,] inputMatrix = problemData.Dataset.ToArray(variables, trainingIndices);
+
+      alglib.dfreport rep;
+      var dForest = CreateRandomForestModel(seed, inputMatrix, nTrees, r, m, 1, out rep);
+
+      rmsError = rep.rmserror;
+      outOfBagRmsError = rep.oobrmserror;
+      avgRelError = rep.avgrelerror;
+      outOfBagAvgRelError = rep.oobavgrelerror;
+
+      return new RandomForestModel(problemData.TargetVariable, dForest, seed, problemData, nTrees, r, m);
+    }
+
+    public static RandomForestModel CreateClassificationModel(IClassificationProblemData problemData, int nTrees, double r, double m, int seed,
+      out double rmsError, out double outOfBagRmsError, out double relClassificationError, out double outOfBagRelClassificationError) {
+      return CreateClassificationModel(problemData, problemData.TrainingIndices, nTrees, r, m, seed,
+        out rmsError, out outOfBagRmsError, out relClassificationError, out outOfBagRelClassificationError);
+    }
+
+    public static RandomForestModel CreateClassificationModel(IClassificationProblemData problemData, IEnumerable<int> trainingIndices, int nTrees, double r, double m, int seed,
+      out double rmsError, out double outOfBagRmsError, out double relClassificationError, out double outOfBagRelClassificationError) {
+
+      var variables = problemData.AllowedInputVariables.Concat(new string[] { problemData.TargetVariable });
+      double[,] inputMatrix = problemData.Dataset.ToArray(variables, trainingIndices);
+
+      var classValues = problemData.ClassValues.ToArray();
+      int nClasses = classValues.Length;
+
+      // map original class values to values [0..nClasses-1]
+      var classIndices = new Dictionary<double, double>();
+      for (int i = 0; i < nClasses; i++) {
+        classIndices[classValues[i]] = i;
+      }
+
+      int nRows = inputMatrix.GetLength(0);
+      int nColumns = inputMatrix.GetLength(1);
+      for (int row = 0; row < nRows; row++) {
+        inputMatrix[row, nColumns - 1] = classIndices[inputMatrix[row, nColumns - 1]];
+      }
+
+      alglib.dfreport rep;
+      var dForest = CreateRandomForestModel(seed, inputMatrix, nTrees, r, m, nClasses, out rep);
+
+      rmsError = rep.rmserror;
+      outOfBagRmsError = rep.oobrmserror;
+      relClassificationError = rep.relclserror;
+      outOfBagRelClassificationError = rep.oobrelclserror;
+
+      return new RandomForestModel(problemData.TargetVariable, dForest, seed, problemData, nTrees, r, m, classValues);
+    }
+
+    private static alglib.decisionforest CreateRandomForestModel(int seed, double[,] inputMatrix, int nTrees, double r, double m, int nClasses, out alglib.dfreport rep) {
+      AssertParameters(r, m);
+      AssertInputMatrix(inputMatrix);
+
+      int info = 0;
+      alglib.math.rndobject = new System.Random(seed);
+      var dForest = new alglib.decisionforest();
+      rep = new alglib.dfreport();
+      int nRows = inputMatrix.GetLength(0);
+      int nColumns = inputMatrix.GetLength(1);
+      int sampleSize = Math.Max((int)Math.Round(r * nRows), 1);
+      int nFeatures = Math.Max((int)Math.Round(m * (nColumns - 1)), 1);
+
+      alglib.dforest.dfbuildinternal(inputMatrix, nRows, nColumns - 1, nClasses, nTrees, sampleSize, nFeatures, alglib.dforest.dfusestrongsplits + alglib.dforest.dfuseevs, ref info, dForest.innerobj, rep.innerobj);
+      if (info != 1) throw new ArgumentException("Error in calculation of random forest model");
+      return dForest;
+    }
+
+    private static void AssertParameters(double r, double m) {
+      if (r <= 0 || r > 1) throw new ArgumentException("The R parameter for random forest modeling must be between 0 and 1.");
+      if (m <= 0 || m > 1) throw new ArgumentException("The M parameter for random forest modeling must be between 0 and 1.");
+    }
+
+    private static void AssertInputMatrix(double[,] inputMatrix) {
+      if (inputMatrix.ContainsNanOrInfinity())
+        throw new NotSupportedException("Random forest modeling does not support NaN or infinity values in the input dataset.");
+    }
+
+    #region persistence for backwards compatibility
+    // when the originalTrainingData is null this means the model was loaded from an old file 
+    // therefore, we cannot use the new persistence mechanism because the original data is not available anymore
+    // in such cases we still store the compete model
+    private bool IsCompatibilityLoaded { get { return originalTrainingData == null; } }
+
+    private string[] allowedInputVariables;
+    [Storable(Name = "allowedInputVariables")]
+    private string[] AllowedInputVariables {
+      get {
+        if (IsCompatibilityLoaded) return allowedInputVariables;
+        else return originalTrainingData.AllowedInputVariables.ToArray();
+      }
+      set { allowedInputVariables = value; }
+    }
+    [Storable]
+    private int RandomForestBufSize {
+      get {
+        if (IsCompatibilityLoaded) return randomForest.innerobj.bufsize;
+        else return 0;
+      }
+      set {
+        randomForest.innerobj.bufsize = value;
+      }
+    }
+    [Storable]
+    private int RandomForestNClasses {
+      get {
+        if (IsCompatibilityLoaded) return randomForest.innerobj.nclasses;
+        else return 0;
+      }
+      set {
+        randomForest.innerobj.nclasses = value;
+      }
+    }
+    [Storable]
+    private int RandomForestNTrees {
+      get {
+        if (IsCompatibilityLoaded) return randomForest.innerobj.ntrees;
+        else return 0;
+      }
+      set {
+        randomForest.innerobj.ntrees = value;
+      }
+    }
+    [Storable]
+    private int RandomForestNVars {
+      get {
+        if (IsCompatibilityLoaded) return randomForest.innerobj.nvars;
+        else return 0;
+      }
+      set {
+        randomForest.innerobj.nvars = value;
+      }
+    }
+    [Storable]
+    private double[] RandomForestTrees {
+      get {
+        if (IsCompatibilityLoaded) return randomForest.innerobj.trees;
+        else return new double[] { };
+      }
+      set {
+        randomForest.innerobj.trees = value;
+      }
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegression.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegression.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegression.cs	(revision 17041)
@@ -0,0 +1,168 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Random forest regression data analysis algorithm.
+  /// </summary>
+  [Item("Random Forest Regression (RF)", "Random forest regression data analysis algorithm (wrapper for ALGLIB).")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 120)]
+  [StorableType("721CE0EB-82AF-4E49-9900-48E1C67B5E53")]
+  public sealed class RandomForestRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+    private const string RandomForestRegressionModelResultName = "Random forest regression solution";
+    private const string NumberOfTreesParameterName = "Number of trees";
+    private const string RParameterName = "R";
+    private const string MParameterName = "M";
+    private const string SeedParameterName = "Seed";
+    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+    private const string CreateSolutionParameterName = "CreateSolution";
+
+    #region parameter properties
+    public IFixedValueParameter<IntValue> NumberOfTreesParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NumberOfTreesParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> RParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[RParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> MParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[MParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> SeedParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> CreateSolutionParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
+    }
+    #endregion
+    #region properties
+    public int NumberOfTrees {
+      get { return NumberOfTreesParameter.Value.Value; }
+      set { NumberOfTreesParameter.Value.Value = value; }
+    }
+    public double R {
+      get { return RParameter.Value.Value; }
+      set { RParameter.Value.Value = value; }
+    }
+    public double M {
+      get { return MParameter.Value.Value; }
+      set { MParameter.Value.Value = value; }
+    }
+    public int Seed {
+      get { return SeedParameter.Value.Value; }
+      set { SeedParameter.Value.Value = value; }
+    }
+    public bool SetSeedRandomly {
+      get { return SetSeedRandomlyParameter.Value.Value; }
+      set { SetSeedRandomlyParameter.Value.Value = value; }
+    }
+    public bool CreateSolution {
+      get { return CreateSolutionParameter.Value.Value; }
+      set { CreateSolutionParameter.Value.Value = value; }
+    }
+    #endregion
+    [StorableConstructor]
+    private RandomForestRegression(StorableConstructorFlag _) : base(_) { }
+    private RandomForestRegression(RandomForestRegression original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public RandomForestRegression()
+      : base() {
+      Parameters.Add(new FixedValueParameter<IntValue>(NumberOfTreesParameterName, "The number of trees in the forest. Should be between 50 and 100", new IntValue(50)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(RParameterName, "The ratio of the training set that will be used in the construction of individual trees (0<r<=1). Should be adjusted depending on the noise level in the dataset in the range from 0.66 (low noise) to 0.05 (high noise). This parameter should be adjusted to achieve good generalization error.", new DoubleValue(0.3)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName, "The ratio of features that will be used in the construction of individual trees (0<m<=1)", new DoubleValue(0.5)));
+      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+      Parameters[CreateSolutionParameterName].Hidden = true;
+
+      Problem = new RegressionProblem();
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
+      if (!Parameters.ContainsKey(MParameterName))
+        Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName, "The ratio of features that will be used in the construction of individual trees (0<m<=1)", new DoubleValue(0.5)));
+      if (!Parameters.ContainsKey(SeedParameterName))
+        Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+      if (!Parameters.ContainsKey((SetSeedRandomlyParameterName)))
+        Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+      if (!Parameters.ContainsKey(CreateSolutionParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+        Parameters[CreateSolutionParameterName].Hidden = true;
+      }
+      #endregion
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RandomForestRegression(this, cloner);
+    }
+
+    #region random forest
+    protected override void Run(CancellationToken cancellationToken) {
+      double rmsError, avgRelError, outOfBagRmsError, outOfBagAvgRelError;
+      if (SetSeedRandomly) Seed = Random.RandomSeedGenerator.GetSeed();
+      var model = CreateRandomForestRegressionModel(Problem.ProblemData, NumberOfTrees, R, M, Seed,
+        out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError);
+
+      Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the random forest regression solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Average relative error", "The average of relative errors of the random forest regression solution on the training set.", new PercentValue(avgRelError)));
+      Results.Add(new Result("Root mean square error (out-of-bag)", "The out-of-bag root of the mean of squared errors of the random forest regression solution.", new DoubleValue(outOfBagRmsError)));
+      Results.Add(new Result("Average relative error (out-of-bag)", "The out-of-bag average of relative errors of the random forest regression solution.", new PercentValue(outOfBagAvgRelError)));
+
+      if (CreateSolution) {
+        var solution = new RandomForestRegressionSolution(model, (IRegressionProblemData)Problem.ProblemData.Clone());
+        Results.Add(new Result(RandomForestRegressionModelResultName, "The random forest regression solution.", solution));
+      }
+    }
+
+    // keep for compatibility with old API
+    public static RandomForestRegressionSolution CreateRandomForestRegressionSolution(IRegressionProblemData problemData, int nTrees, double r, double m, int seed,
+      out double rmsError, out double avgRelError, out double outOfBagRmsError, out double outOfBagAvgRelError) {
+      var model = CreateRandomForestRegressionModel(problemData, nTrees, r, m, seed,
+        out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError);
+      return new RandomForestRegressionSolution(model, (IRegressionProblemData)problemData.Clone());
+    }
+
+    public static RandomForestModel CreateRandomForestRegressionModel(IRegressionProblemData problemData, int nTrees,
+      double r, double m, int seed,
+      out double rmsError, out double avgRelError, out double outOfBagRmsError, out double outOfBagAvgRelError) {
+      return RandomForestModel.CreateRegressionModel(problemData, nTrees, r, m, seed,
+        rmsError: out rmsError, avgRelError: out avgRelError, outOfBagRmsError: out outOfBagRmsError, outOfBagAvgRelError: out outOfBagAvgRelError);
+    }
+
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegressionSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegressionSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegressionSolution.cs	(revision 17041)
@@ -0,0 +1,57 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a random forest solution for a regression problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("RandomForestRegressionSolution", "Represents a random forest solution for a regression problem which can be visualized in the GUI.")]
+  [StorableType("29F87380-B32D-4BF6-B7F8-653B8AFFAC34")]
+  public sealed class RandomForestRegressionSolution : ConfidenceRegressionSolution, IRandomForestRegressionSolution {
+
+    public new IRandomForestModel Model {
+      get { return (IRandomForestModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    public int NumberOfTrees {
+      get { return Model.NumberOfTrees; }
+    }
+
+    [StorableConstructor]
+    private RandomForestRegressionSolution(StorableConstructorFlag _) : base(_) { }
+    private RandomForestRegressionSolution(RandomForestRegressionSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public RandomForestRegressionSolution(IRandomForestModel randomForestModel, IRegressionProblemData problemData)
+      : base(randomForestModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RandomForestRegressionSolution(this, cloner);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestUtil.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestUtil.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestUtil.cs	(revision 17041)
@@ -0,0 +1,357 @@
+﻿#region License Information
+
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Linq.Expressions;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Random;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("RFParameter", "A random forest parameter collection")]
+  [StorableType("40E482DA-63C5-4D39-97C7-63701CF1D021")]
+  public class RFParameter : ParameterCollection {
+    public RFParameter() {
+      base.Add(new FixedValueParameter<IntValue>("N", "The number of random forest trees", new IntValue(50)));
+      base.Add(new FixedValueParameter<DoubleValue>("M", "The ratio of features that will be used in the construction of individual trees (0<m<=1)", new DoubleValue(0.1)));
+      base.Add(new FixedValueParameter<DoubleValue>("R", "The ratio of the training set that will be used in the construction of individual trees (0<r<=1)", new DoubleValue(0.1)));
+    }
+
+    [StorableConstructor]
+    protected RFParameter(StorableConstructorFlag _) : base(_) {
+    }
+
+    protected RFParameter(RFParameter original, Cloner cloner)
+      : base(original, cloner) {
+      this.N = original.N;
+      this.R = original.R;
+      this.M = original.M;
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RFParameter(this, cloner);
+    }
+
+    private IFixedValueParameter<IntValue> NParameter {
+      get { return (IFixedValueParameter<IntValue>)base["N"]; }
+    }
+
+    private IFixedValueParameter<DoubleValue> RParameter {
+      get { return (IFixedValueParameter<DoubleValue>)base["R"]; }
+    }
+
+    private IFixedValueParameter<DoubleValue> MParameter {
+      get { return (IFixedValueParameter<DoubleValue>)base["M"]; }
+    }
+
+    public int N {
+      get { return NParameter.Value.Value; }
+      set { NParameter.Value.Value = value; }
+    }
+
+    public double R {
+      get { return RParameter.Value.Value; }
+      set { RParameter.Value.Value = value; }
+    }
+
+    public double M {
+      get { return MParameter.Value.Value; }
+      set { MParameter.Value.Value = value; }
+    }
+  }
+
+  public static class RandomForestUtil {
+    private static void CrossValidate(IRegressionProblemData problemData, Tuple<IEnumerable<int>, IEnumerable<int>>[] partitions, int nTrees, double r, double m, int seed, out double avgTestMse) {
+      avgTestMse = 0;
+      var ds = problemData.Dataset;
+      var targetVariable = GetTargetVariableName(problemData);
+      foreach (var tuple in partitions) {
+        double rmsError, avgRelError, outOfBagAvgRelError, outOfBagRmsError;
+        var trainingRandomForestPartition = tuple.Item1;
+        var testRandomForestPartition = tuple.Item2;
+        var model = RandomForestModel.CreateRegressionModel(problemData, trainingRandomForestPartition, nTrees, r, m, seed, out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError);
+        var estimatedValues = model.GetEstimatedValues(ds, testRandomForestPartition);
+        var targetValues = ds.GetDoubleValues(targetVariable, testRandomForestPartition);
+        OnlineCalculatorError calculatorError;
+        double mse = OnlineMeanSquaredErrorCalculator.Calculate(estimatedValues, targetValues, out calculatorError);
+        if (calculatorError != OnlineCalculatorError.None)
+          mse = double.NaN;
+        avgTestMse += mse;
+      }
+      avgTestMse /= partitions.Length;
+    }
+
+    private static void CrossValidate(IClassificationProblemData problemData, Tuple<IEnumerable<int>, IEnumerable<int>>[] partitions, int nTrees, double r, double m, int seed, out double avgTestAccuracy) {
+      avgTestAccuracy = 0;
+      var ds = problemData.Dataset;
+      var targetVariable = GetTargetVariableName(problemData);
+      foreach (var tuple in partitions) {
+        double rmsError, avgRelError, outOfBagAvgRelError, outOfBagRmsError;
+        var trainingRandomForestPartition = tuple.Item1;
+        var testRandomForestPartition = tuple.Item2;
+        var model = RandomForestModel.CreateClassificationModel(problemData, trainingRandomForestPartition, nTrees, r, m, seed, out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError);
+        var estimatedValues = model.GetEstimatedClassValues(ds, testRandomForestPartition);
+        var targetValues = ds.GetDoubleValues(targetVariable, testRandomForestPartition);
+        OnlineCalculatorError calculatorError;
+        double accuracy = OnlineAccuracyCalculator.Calculate(estimatedValues, targetValues, out calculatorError);
+        if (calculatorError != OnlineCalculatorError.None)
+          accuracy = double.NaN;
+        avgTestAccuracy += accuracy;
+      }
+      avgTestAccuracy /= partitions.Length;
+    }
+
+    /// <summary>
+    /// Grid search without crossvalidation (since for random forests the out-of-bag estimate is unbiased)
+    /// </summary>
+    /// <param name="problemData">The regression problem data</param>
+    /// <param name="parameterRanges">The ranges for each parameter in the grid search</param>
+    /// <param name="seed">The random seed (required by the random forest model)</param>
+    /// <param name="maxDegreeOfParallelism">The maximum allowed number of threads (to parallelize the grid search)</param>
+    public static RFParameter GridSearch(IRegressionProblemData problemData, Dictionary<string, IEnumerable<double>> parameterRanges, int seed = 12345, int maxDegreeOfParallelism = 1) {
+      var setters = parameterRanges.Keys.Select(GenerateSetter).ToList();
+      var crossProduct = parameterRanges.Values.CartesianProduct();
+      double bestOutOfBagRmsError = double.MaxValue;
+      RFParameter bestParameters = new RFParameter();
+
+      var locker = new object();
+      Parallel.ForEach(crossProduct, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, parameterCombination => {
+        var parameterValues = parameterCombination.ToList();
+        var parameters = new RFParameter();
+        for (int i = 0; i < setters.Count; ++i) { setters[i](parameters, parameterValues[i]); }
+        double rmsError, outOfBagRmsError, avgRelError, outOfBagAvgRelError;
+        RandomForestModel.CreateRegressionModel(problemData, problemData.TrainingIndices, parameters.N, parameters.R, parameters.M, seed, out rmsError, out outOfBagRmsError, out avgRelError, out outOfBagAvgRelError);
+
+        lock (locker) {
+          if (bestOutOfBagRmsError > outOfBagRmsError) {
+            bestOutOfBagRmsError = outOfBagRmsError;
+            bestParameters = (RFParameter)parameters.Clone();
+          }
+        }
+      });
+      return bestParameters;
+    }
+
+    /// <summary>
+    /// Grid search without crossvalidation (since for random forests the out-of-bag estimate is unbiased)
+    /// </summary>
+    /// <param name="problemData">The classification problem data</param>
+    /// <param name="parameterRanges">The ranges for each parameter in the grid search</param>
+    /// <param name="seed">The random seed (required by the random forest model)</param>
+    /// <param name="maxDegreeOfParallelism">The maximum allowed number of threads (to parallelize the grid search)</param>
+    public static RFParameter GridSearch(IClassificationProblemData problemData, Dictionary<string, IEnumerable<double>> parameterRanges, int seed = 12345, int maxDegreeOfParallelism = 1) {
+      var setters = parameterRanges.Keys.Select(GenerateSetter).ToList();
+      var crossProduct = parameterRanges.Values.CartesianProduct();
+
+      double bestOutOfBagRmsError = double.MaxValue;
+      RFParameter bestParameters = new RFParameter();
+
+      var locker = new object();
+      Parallel.ForEach(crossProduct, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, parameterCombination => {
+        var parameterValues = parameterCombination.ToList();
+        var parameters = new RFParameter();
+        for (int i = 0; i < setters.Count; ++i) { setters[i](parameters, parameterValues[i]); }
+        double rmsError, outOfBagRmsError, avgRelError, outOfBagAvgRelError;
+        RandomForestModel.CreateClassificationModel(problemData, problemData.TrainingIndices, parameters.N, parameters.R, parameters.M, seed,
+                                                                out rmsError, out outOfBagRmsError, out avgRelError, out outOfBagAvgRelError);
+
+        lock (locker) {
+          if (bestOutOfBagRmsError > outOfBagRmsError) {
+            bestOutOfBagRmsError = outOfBagRmsError;
+            bestParameters = (RFParameter)parameters.Clone();
+          }
+        }
+      });
+      return bestParameters;
+    }
+
+    /// <summary>
+    /// Grid search with crossvalidation
+    /// </summary>
+    /// <param name="problemData">The regression problem data</param>
+    /// <param name="numberOfFolds">The number of folds for crossvalidation</param>
+    /// <param name="shuffleFolds">Specifies whether the folds should be shuffled</param>
+    /// <param name="parameterRanges">The ranges for each parameter in the grid search</param>
+    /// <param name="seed">The random seed (required by the random forest model)</param>
+    /// <param name="maxDegreeOfParallelism">The maximum allowed number of threads (to parallelize the grid search)</param>
+    /// <returns>The best parameter values found by the grid search</returns>
+    public static RFParameter GridSearch(IRegressionProblemData problemData, int numberOfFolds, bool shuffleFolds, Dictionary<string, IEnumerable<double>> parameterRanges, int seed = 12345, int maxDegreeOfParallelism = 1) {
+      DoubleValue mse = new DoubleValue(Double.MaxValue);
+      RFParameter bestParameter = new RFParameter();
+
+      var setters = parameterRanges.Keys.Select(GenerateSetter).ToList();
+      var partitions = GenerateRandomForestPartitions(problemData, numberOfFolds);
+      var crossProduct = parameterRanges.Values.CartesianProduct();
+
+      var locker = new object();
+      Parallel.ForEach(crossProduct, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, parameterCombination => {
+        var parameterValues = parameterCombination.ToList();
+        double testMSE;
+        var parameters = new RFParameter();
+        for (int i = 0; i < setters.Count; ++i) {
+          setters[i](parameters, parameterValues[i]);
+        }
+        CrossValidate(problemData, partitions, parameters.N, parameters.R, parameters.M, seed, out testMSE);
+
+        lock (locker) {
+          if (testMSE < mse.Value) {
+            mse.Value = testMSE;
+            bestParameter = (RFParameter)parameters.Clone();
+          }
+        }
+      });
+      return bestParameter;
+    }
+
+    /// <summary>
+    /// Grid search with crossvalidation
+    /// </summary>
+    /// <param name="problemData">The classification problem data</param>
+    /// <param name="numberOfFolds">The number of folds for crossvalidation</param>
+    /// <param name="shuffleFolds">Specifies whether the folds should be shuffled</param>
+    /// <param name="parameterRanges">The ranges for each parameter in the grid search</param>
+    /// <param name="seed">The random seed (for shuffling)</param>
+    /// <param name="maxDegreeOfParallelism">The maximum allowed number of threads (to parallelize the grid search)</param>
+    public static RFParameter GridSearch(IClassificationProblemData problemData, int numberOfFolds, bool shuffleFolds, Dictionary<string, IEnumerable<double>> parameterRanges, int seed = 12345, int maxDegreeOfParallelism = 1) {
+      DoubleValue accuracy = new DoubleValue(0);
+      RFParameter bestParameter = new RFParameter();
+
+      var setters = parameterRanges.Keys.Select(GenerateSetter).ToList();
+      var crossProduct = parameterRanges.Values.CartesianProduct();
+      var partitions = GenerateRandomForestPartitions(problemData, numberOfFolds, shuffleFolds);
+
+      var locker = new object();
+      Parallel.ForEach(crossProduct, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, parameterCombination => {
+        var parameterValues = parameterCombination.ToList();
+        double testAccuracy;
+        var parameters = new RFParameter();
+        for (int i = 0; i < setters.Count; ++i) {
+          setters[i](parameters, parameterValues[i]);
+        }
+        CrossValidate(problemData, partitions, parameters.N, parameters.R, parameters.M, seed, out testAccuracy);
+
+        lock (locker) {
+          if (testAccuracy > accuracy.Value) {
+            accuracy.Value = testAccuracy;
+            bestParameter = (RFParameter)parameters.Clone();
+          }
+        }
+      });
+      return bestParameter;
+    }
+
+    private static Tuple<IEnumerable<int>, IEnumerable<int>>[] GenerateRandomForestPartitions(IDataAnalysisProblemData problemData, int numberOfFolds, bool shuffleFolds = false) {
+      var folds = GenerateFolds(problemData, numberOfFolds, shuffleFolds).ToList();
+      var partitions = new Tuple<IEnumerable<int>, IEnumerable<int>>[numberOfFolds];
+
+      for (int i = 0; i < numberOfFolds; ++i) {
+        int p = i; // avoid "access to modified closure" warning
+        var trainingRows = folds.SelectMany((par, j) => j != p ? par : Enumerable.Empty<int>());
+        var testRows = folds[i];
+        partitions[i] = new Tuple<IEnumerable<int>, IEnumerable<int>>(trainingRows, testRows);
+      }
+      return partitions;
+    }
+
+    public static IEnumerable<IEnumerable<int>> GenerateFolds(IDataAnalysisProblemData problemData, int numberOfFolds, bool shuffleFolds = false) {
+      var random = new MersenneTwister((uint)Environment.TickCount);
+      if (problemData is IRegressionProblemData) {
+        var trainingIndices = shuffleFolds ? problemData.TrainingIndices.OrderBy(x => random.Next()) : problemData.TrainingIndices;
+        return GenerateFolds(trainingIndices, problemData.TrainingPartition.Size, numberOfFolds);
+      }
+      if (problemData is IClassificationProblemData) {
+        // when shuffle is enabled do stratified folds generation, some folds may have zero elements 
+        // otherwise, generate folds normally
+        return shuffleFolds ? GenerateFoldsStratified(problemData as IClassificationProblemData, numberOfFolds, random) : GenerateFolds(problemData.TrainingIndices, problemData.TrainingPartition.Size, numberOfFolds);
+      }
+      throw new ArgumentException("Problem data is neither regression or classification problem data.");
+    }
+
+    /// <summary>
+    /// Stratified fold generation from classification data. Stratification means that we ensure the same distribution of class labels for each fold.
+    /// The samples are grouped by class label and each group is split into @numberOfFolds parts. The final folds are formed from the joining of 
+    /// the corresponding parts from each class label.
+    /// </summary>
+    /// <param name="problemData">The classification problem data.</param>
+    /// <param name="numberOfFolds">The number of folds in which to split the data.</param>
+    /// <param name="random">The random generator used to shuffle the folds.</param>
+    /// <returns>An enumerable sequece of folds, where a fold is represented by a sequence of row indices.</returns>
+    private static IEnumerable<IEnumerable<int>> GenerateFoldsStratified(IClassificationProblemData problemData, int numberOfFolds, IRandom random) {
+      var values = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+      var valuesIndices = problemData.TrainingIndices.Zip(values, (i, v) => new { Index = i, Value = v }).ToList();
+      IEnumerable<IEnumerable<IEnumerable<int>>> foldsByClass = valuesIndices.GroupBy(x => x.Value, x => x.Index).Select(g => GenerateFolds(g, g.Count(), numberOfFolds));
+      var enumerators = foldsByClass.Select(f => f.GetEnumerator()).ToList();
+      while (enumerators.All(e => e.MoveNext())) {
+        yield return enumerators.SelectMany(e => e.Current).OrderBy(x => random.Next()).ToList();
+      }
+    }
+
+    private static IEnumerable<IEnumerable<T>> GenerateFolds<T>(IEnumerable<T> values, int valuesCount, int numberOfFolds) {
+      // if number of folds is greater than the number of values, some empty folds will be returned
+      if (valuesCount < numberOfFolds) {
+        for (int i = 0; i < numberOfFolds; ++i)
+          yield return i < valuesCount ? values.Skip(i).Take(1) : Enumerable.Empty<T>();
+      } else {
+        int f = valuesCount / numberOfFolds, r = valuesCount % numberOfFolds; // number of folds rounded to integer and remainder
+        int start = 0, end = f;
+        for (int i = 0; i < numberOfFolds; ++i) {
+          if (r > 0) {
+            ++end;
+            --r;
+          }
+          yield return values.Skip(start).Take(end - start);
+          start = end;
+          end += f;
+        }
+      }
+    }
+
+    private static Action<RFParameter, double> GenerateSetter(string field) {
+      var targetExp = Expression.Parameter(typeof(RFParameter));
+      var valueExp = Expression.Parameter(typeof(double));
+      var fieldExp = Expression.Property(targetExp, field);
+      var assignExp = Expression.Assign(fieldExp, Expression.Convert(valueExp, fieldExp.Type));
+      var setter = Expression.Lambda<Action<RFParameter, double>>(assignExp, targetExp, valueExp).Compile();
+      return setter;
+    }
+
+    private static string GetTargetVariableName(IDataAnalysisProblemData problemData) {
+      var regressionProblemData = problemData as IRegressionProblemData;
+      var classificationProblemData = problemData as IClassificationProblemData;
+
+      if (regressionProblemData != null)
+        return regressionProblemData.TargetVariable;
+      if (classificationProblemData != null)
+        return classificationProblemData.TargetVariable;
+
+      throw new ArgumentException("Problem data is neither regression or classification problem data.");
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassification.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassification.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassification.cs	(revision 17041)
@@ -0,0 +1,268 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+using LibSVM;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Support vector machine classification data analysis algorithm.
+  /// </summary>
+  [Item("Support Vector Classification (SVM)", "Support vector machine classification data analysis algorithm (wrapper for libSVM).")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 110)]
+  [StorableType("F15289E4-B648-4A92-AB01-14D769A33967")]
+  public sealed class SupportVectorClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+    private const string SvmTypeParameterName = "SvmType";
+    private const string KernelTypeParameterName = "KernelType";
+    private const string CostParameterName = "Cost";
+    private const string NuParameterName = "Nu";
+    private const string GammaParameterName = "Gamma";
+    private const string DegreeParameterName = "Degree";
+    private const string CreateSolutionParameterName = "CreateSolution";
+
+    #region parameter properties
+    public IConstrainedValueParameter<StringValue> SvmTypeParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[SvmTypeParameterName]; }
+    }
+    public IConstrainedValueParameter<StringValue> KernelTypeParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[KernelTypeParameterName]; }
+    }
+    public IValueParameter<DoubleValue> NuParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters[NuParameterName]; }
+    }
+    public IValueParameter<DoubleValue> CostParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters[CostParameterName]; }
+    }
+    public IValueParameter<DoubleValue> GammaParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters[GammaParameterName]; }
+    }
+    public IValueParameter<IntValue> DegreeParameter {
+      get { return (IValueParameter<IntValue>)Parameters[DegreeParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> CreateSolutionParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
+    }
+    #endregion
+    #region properties
+    public StringValue SvmType {
+      get { return SvmTypeParameter.Value; }
+      set { SvmTypeParameter.Value = value; }
+    }
+    public StringValue KernelType {
+      get { return KernelTypeParameter.Value; }
+      set { KernelTypeParameter.Value = value; }
+    }
+    public DoubleValue Nu {
+      get { return NuParameter.Value; }
+    }
+    public DoubleValue Cost {
+      get { return CostParameter.Value; }
+    }
+    public DoubleValue Gamma {
+      get { return GammaParameter.Value; }
+    }
+    public IntValue Degree {
+      get { return DegreeParameter.Value; }
+    }
+    public bool CreateSolution {
+      get { return CreateSolutionParameter.Value.Value; }
+      set { CreateSolutionParameter.Value.Value = value; }
+    }
+    #endregion
+    [StorableConstructor]
+    private SupportVectorClassification(StorableConstructorFlag _) : base(_) { }
+    private SupportVectorClassification(SupportVectorClassification original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public SupportVectorClassification()
+      : base() {
+      Problem = new ClassificationProblem();
+
+      List<StringValue> svrTypes = (from type in new List<string> { "NU_SVC", "C_SVC" }
+                                    select new StringValue(type).AsReadOnly())
+                                   .ToList();
+      ItemSet<StringValue> svrTypeSet = new ItemSet<StringValue>(svrTypes);
+      List<StringValue> kernelTypes = (from type in new List<string> { "LINEAR", "POLY", "SIGMOID", "RBF" }
+                                       select new StringValue(type).AsReadOnly())
+                                   .ToList();
+      ItemSet<StringValue> kernelTypeSet = new ItemSet<StringValue>(kernelTypes);
+      Parameters.Add(new ConstrainedValueParameter<StringValue>(SvmTypeParameterName, "The type of SVM to use.", svrTypeSet, svrTypes[0]));
+      Parameters.Add(new ConstrainedValueParameter<StringValue>(KernelTypeParameterName, "The kernel type to use for the SVM.", kernelTypeSet, kernelTypes[3]));
+      Parameters.Add(new ValueParameter<DoubleValue>(NuParameterName, "The value of the nu parameter nu-SVC.", new DoubleValue(0.5)));
+      Parameters.Add(new ValueParameter<DoubleValue>(CostParameterName, "The value of the C (cost) parameter of C-SVC.", new DoubleValue(1.0)));
+      Parameters.Add(new ValueParameter<DoubleValue>(GammaParameterName, "The value of the gamma parameter in the kernel function.", new DoubleValue(1.0)));
+      Parameters.Add(new ValueParameter<IntValue>(DegreeParameterName, "The degree parameter for the polynomial kernel function.", new IntValue(3)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+      Parameters[CreateSolutionParameterName].Hidden = true;
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      #region backwards compatibility (change with 3.4)
+      if (!Parameters.ContainsKey(DegreeParameterName)) {
+        Parameters.Add(new ValueParameter<IntValue>(DegreeParameterName,
+          "The degree parameter for the polynomial kernel function.", new IntValue(3)));
+      }
+      if (!Parameters.ContainsKey(CreateSolutionParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName,
+          "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+        Parameters[CreateSolutionParameterName].Hidden = true;
+      }
+      #endregion
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new SupportVectorClassification(this, cloner);
+    }
+
+    #region support vector classification
+    protected override void Run(CancellationToken cancellationToken) {
+      IClassificationProblemData problemData = Problem.ProblemData;
+      IEnumerable<string> selectedInputVariables = problemData.AllowedInputVariables;
+      int nSv;
+      ISupportVectorMachineModel model;
+
+      Run(problemData, selectedInputVariables, GetSvmType(SvmType.Value), GetKernelType(KernelType.Value), Cost.Value, Nu.Value, Gamma.Value, Degree.Value, out model, out nSv);
+
+      if (CreateSolution) {
+        var solution = new SupportVectorClassificationSolution((SupportVectorMachineModel)model, (IClassificationProblemData)problemData.Clone());
+        Results.Add(new Result("Support vector classification solution", "The support vector classification solution.",
+          solution));
+      }
+
+      {
+        // calculate classification metrics
+        // calculate regression model metrics 
+        var ds = problemData.Dataset;
+        var trainRows = problemData.TrainingIndices;
+        var testRows = problemData.TestIndices;
+        var yTrain = ds.GetDoubleValues(problemData.TargetVariable, trainRows);
+        var yTest = ds.GetDoubleValues(problemData.TargetVariable, testRows);
+        var yPredTrain = model.GetEstimatedClassValues(ds, trainRows);
+        var yPredTest = model.GetEstimatedClassValues(ds, testRows);
+
+        OnlineCalculatorError error;
+        var trainAccuracy = OnlineAccuracyCalculator.Calculate(yPredTrain, yTrain, out error);
+        if (error != OnlineCalculatorError.None) trainAccuracy = double.MaxValue;
+        var testAccuracy = OnlineAccuracyCalculator.Calculate(yPredTest, yTest, out error);
+        if (error != OnlineCalculatorError.None) testAccuracy = double.MaxValue;
+
+        Results.Add(new Result("Accuracy (training)", "The mean of squared errors of the SVR solution on the training partition.", new DoubleValue(trainAccuracy)));
+        Results.Add(new Result("Accuracy (test)", "The mean of squared errors of the SVR solution on the test partition.", new DoubleValue(testAccuracy)));
+
+        Results.Add(new Result("Number of support vectors", "The number of support vectors of the SVR solution.",
+          new IntValue(nSv)));
+      }
+    }
+
+    public static SupportVectorClassificationSolution CreateSupportVectorClassificationSolution(IClassificationProblemData problemData, IEnumerable<string> allowedInputVariables,
+      string svmType, string kernelType, double cost, double nu, double gamma, int degree, out double trainingAccuracy, out double testAccuracy, out int nSv) {
+      return CreateSupportVectorClassificationSolution(problemData, allowedInputVariables, GetSvmType(svmType), GetKernelType(kernelType), cost, nu, gamma, degree,
+        out trainingAccuracy, out testAccuracy, out nSv);
+    }
+
+    // BackwardsCompatibility3.4
+    #region Backwards compatible code, remove with 3.5
+    public static SupportVectorClassificationSolution CreateSupportVectorClassificationSolution(IClassificationProblemData problemData, IEnumerable<string> allowedInputVariables,
+      int svmType, int kernelType, double cost, double nu, double gamma, int degree, out double trainingAccuracy, out double testAccuracy, out int nSv) {
+
+      ISupportVectorMachineModel model;
+      Run(problemData, allowedInputVariables, svmType, kernelType, cost, nu, gamma, degree, out model, out nSv);
+      var solution = new SupportVectorClassificationSolution((SupportVectorMachineModel)model, (IClassificationProblemData)problemData.Clone());
+
+      trainingAccuracy = solution.TrainingAccuracy;
+      testAccuracy = solution.TestAccuracy;
+
+      return solution;
+    }
+
+    #endregion
+
+    public static void Run(IClassificationProblemData problemData, IEnumerable<string> allowedInputVariables,
+      int svmType, int kernelType, double cost, double nu, double gamma, int degree,
+      out ISupportVectorMachineModel model, out int nSv) {
+      var dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+
+      svm_parameter parameter = new svm_parameter {
+        svm_type = svmType,
+        kernel_type = kernelType,
+        C = cost,
+        nu = nu,
+        gamma = gamma,
+        cache_size = 500,
+        probability = 0,
+        eps = 0.001,
+        degree = degree,
+        shrinking = 1,
+        coef0 = 0
+      };
+
+      var weightLabels = new List<int>();
+      var weights = new List<double>();
+      foreach (double c in problemData.ClassValues) {
+        double wSum = 0.0;
+        foreach (double otherClass in problemData.ClassValues) {
+          if (!c.IsAlmost(otherClass)) {
+            wSum += problemData.GetClassificationPenalty(c, otherClass);
+          }
+        }
+        weightLabels.Add((int)c);
+        weights.Add(wSum);
+      }
+      parameter.weight_label = weightLabels.ToArray();
+      parameter.weight = weights.ToArray();
+
+      svm_problem problem = SupportVectorMachineUtil.CreateSvmProblem(dataset, targetVariable, allowedInputVariables, rows);
+      RangeTransform rangeTransform = RangeTransform.Compute(problem);
+      svm_problem scaledProblem = rangeTransform.Scale(problem);
+      var svmModel = svm.svm_train(scaledProblem, parameter);
+      nSv = svmModel.SV.Length;
+
+      model = new SupportVectorMachineModel(svmModel, rangeTransform, targetVariable, allowedInputVariables, problemData.ClassValues);
+    }
+
+    private static int GetSvmType(string svmType) {
+      if (svmType == "NU_SVC") return svm_parameter.NU_SVC;
+      if (svmType == "C_SVC") return svm_parameter.C_SVC;
+      throw new ArgumentException("Unknown SVM type");
+    }
+
+    private static int GetKernelType(string kernelType) {
+      if (kernelType == "LINEAR") return svm_parameter.LINEAR;
+      if (kernelType == "POLY") return svm_parameter.POLY;
+      if (kernelType == "SIGMOID") return svm_parameter.SIGMOID;
+      if (kernelType == "RBF") return svm_parameter.RBF;
+      throw new ArgumentException("Unknown kernel type");
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassificationSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassificationSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassificationSolution.cs	(revision 17041)
@@ -0,0 +1,53 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a support vector solution for a classification problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("SupportVectorClassificationSolution", "Represents a support vector solution for a classification problem which can be visualized in the GUI.")]
+  [StorableType("81EC96ED-A900-4517-B4C0-0B4EA3ABB94A")]
+  public sealed class SupportVectorClassificationSolution : ClassificationSolution, ISupportVectorMachineSolution {
+
+    public new ISupportVectorMachineModel Model {
+      get { return (ISupportVectorMachineModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private SupportVectorClassificationSolution(StorableConstructorFlag _) : base(_) { }
+    private SupportVectorClassificationSolution(SupportVectorClassificationSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public SupportVectorClassificationSolution(SupportVectorMachineModel model, IClassificationProblemData problemData)
+      : base(model, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new SupportVectorClassificationSolution(this, cloner);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineModel.cs	(revision 17041)
@@ -0,0 +1,228 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+using LibSVM;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a support vector machine model.
+  /// </summary>
+  [StorableType("669B0522-A27B-4DA6-B655-27C374D88C95")]
+  [Item("SupportVectorMachineModel", "Represents a support vector machine model.")]
+  public sealed class SupportVectorMachineModel : ClassificationModel, ISupportVectorMachineModel {
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return allowedInputVariables; }
+    }
+
+
+    private svm_model model;
+    /// <summary>
+    /// Gets or sets the SVM model.
+    /// </summary>
+    public svm_model Model {
+      get { return model; }
+      set {
+        if (value != model) {
+          if (value == null) throw new ArgumentNullException();
+          model = value;
+          OnChanged(EventArgs.Empty);
+        }
+      }
+    }
+
+    /// <summary>
+    /// Gets or sets the range transformation for the model.
+    /// </summary>
+    private RangeTransform rangeTransform;
+    public RangeTransform RangeTransform {
+      get { return rangeTransform; }
+      set {
+        if (value != rangeTransform) {
+          if (value == null) throw new ArgumentNullException();
+          rangeTransform = value;
+          OnChanged(EventArgs.Empty);
+        }
+      }
+    }
+
+    public Dataset SupportVectors {
+      get {
+        var data = new double[Model.sv_coef.Length, allowedInputVariables.Count()];
+        for (int i = 0; i < Model.sv_coef.Length; i++) {
+          var sv = Model.SV[i];
+          for (int j = 0; j < sv.Length; j++) {
+            data[i, j] = sv[j].value;
+          }
+        }
+        return new Dataset(allowedInputVariables, data);
+      }
+    }
+
+    [Storable]
+    private string[] allowedInputVariables;
+    [Storable]
+    private double[] classValues; // only for SVM classification models
+
+    [StorableConstructor]
+    private SupportVectorMachineModel(StorableConstructorFlag _) : base(_) { }
+    private SupportVectorMachineModel(SupportVectorMachineModel original, Cloner cloner)
+      : base(original, cloner) {
+      // only using a shallow copy here! (gkronber)
+      this.model = original.model;
+      this.rangeTransform = original.rangeTransform;
+      this.allowedInputVariables = (string[])original.allowedInputVariables.Clone();
+      if (original.classValues != null)
+        this.classValues = (double[])original.classValues.Clone();
+    }
+    public SupportVectorMachineModel(svm_model model, RangeTransform rangeTransform, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<double> classValues)
+      : this(model, rangeTransform, targetVariable, allowedInputVariables) {
+      this.classValues = classValues.ToArray();
+    }
+    public SupportVectorMachineModel(svm_model model, RangeTransform rangeTransform, string targetVariable, IEnumerable<string> allowedInputVariables)
+      : base(targetVariable) {
+      this.name = ItemName;
+      this.description = ItemDescription;
+      this.model = model;
+      this.rangeTransform = rangeTransform;
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new SupportVectorMachineModel(this, cloner);
+    }
+
+    #region IRegressionModel Members
+    public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      return GetEstimatedValuesHelper(dataset, rows);
+    }
+    public IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new SupportVectorRegressionSolution(this, new RegressionProblemData(problemData));
+    }
+
+    public bool IsProblemDataCompatible(IRegressionProblemData problemData, out string errorMessage) {
+      return RegressionModel.IsProblemDataCompatible(this, problemData, out errorMessage);
+    }
+    #endregion
+
+    public override bool IsProblemDataCompatible(IDataAnalysisProblemData problemData, out string errorMessage) {
+      if (problemData == null) throw new ArgumentNullException("problemData", "The provided problemData is null.");
+
+      var regressionProblemData = problemData as IRegressionProblemData;
+      if (regressionProblemData != null)
+        return IsProblemDataCompatible(regressionProblemData, out errorMessage);
+
+      var classificationProblemData = problemData as IClassificationProblemData;
+      if (classificationProblemData != null)
+        return IsProblemDataCompatible(classificationProblemData, out errorMessage);
+
+      throw new ArgumentException("The problem data is not compatible with this SVM. Instead a " + problemData.GetType().GetPrettyName() + " was provided.", "problemData");
+    }
+
+    #region IClassificationModel Members
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+      if (classValues == null) throw new NotSupportedException();
+      // return the original class value instead of the predicted value of the model
+      // svm classification only works for integer classes
+      foreach (var estimated in GetEstimatedValuesHelper(dataset, rows)) {
+        // find closest class
+        double bestDist = double.MaxValue;
+        double bestClass = -1;
+        for (int i = 0; i < classValues.Length; i++) {
+          double d = Math.Abs(estimated - classValues[i]);
+          if (d < bestDist) {
+            bestDist = d;
+            bestClass = classValues[i];
+            if (d.IsAlmost(0.0)) break; // exact match no need to look further
+          }
+        }
+        yield return bestClass;
+      }
+    }
+
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new SupportVectorClassificationSolution(this, new ClassificationProblemData(problemData));
+    }
+    #endregion
+
+    private IEnumerable<double> GetEstimatedValuesHelper(IDataset dataset, IEnumerable<int> rows) {
+      // calculate predictions for the currently requested rows
+      svm_problem problem = SupportVectorMachineUtil.CreateSvmProblem(dataset, allowedInputVariables, rows);
+      svm_problem scaledProblem = rangeTransform.Scale(problem);
+
+      for (int i = 0; i < problem.l; i++) {
+        yield return svm.svm_predict(Model, scaledProblem.x[i]);
+      }
+    }
+
+    #region events
+    public event EventHandler Changed;
+    private void OnChanged(EventArgs e) {
+      var handlers = Changed;
+      if (handlers != null)
+        handlers(this, e);
+    }
+    #endregion
+
+    #region persistence
+    [Storable]
+    private string ModelAsString {
+      get {
+        using (MemoryStream stream = new MemoryStream()) {
+          svm.svm_save_model(new StreamWriter(stream), Model);
+          stream.Seek(0, System.IO.SeekOrigin.Begin);
+          StreamReader reader = new StreamReader(stream);
+          return reader.ReadToEnd();
+        }
+      }
+      set {
+        using (MemoryStream stream = new MemoryStream(Encoding.ASCII.GetBytes(value))) {
+          model = svm.svm_load_model(new StreamReader(stream));
+        }
+      }
+    }
+    [Storable]
+    private string RangeTransformAsString {
+      get {
+        using (MemoryStream stream = new MemoryStream()) {
+          RangeTransform.Write(stream, RangeTransform);
+          stream.Seek(0, System.IO.SeekOrigin.Begin);
+          StreamReader reader = new StreamReader(stream);
+          return reader.ReadToEnd();
+        }
+      }
+      set {
+        using (MemoryStream stream = new MemoryStream(Encoding.ASCII.GetBytes(value))) {
+          RangeTransform = RangeTransform.Read(stream);
+        }
+      }
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineUtil.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineUtil.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineUtil.cs	(revision 17041)
@@ -0,0 +1,251 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Linq.Expressions;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Random;
+using LibSVM;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public class SupportVectorMachineUtil {
+    /// <summary>
+    /// Transforms <paramref name="dataset"/> into a data structure as needed by libSVM.
+    /// </summary>
+    /// <param name="dataset">The source dataset</param>
+    /// <param name="targetVariable">The target variable</param>
+    /// <param name="inputVariables">The selected input variables to include in the svm_problem.</param>
+    /// <param name="rowIndices">The rows of the dataset that should be contained in the resulting SVM-problem</param>
+    /// <returns>A problem data type that can be used to train a support vector machine.</returns>
+    public static svm_problem CreateSvmProblem(IDataset dataset, string targetVariable, IEnumerable<string> inputVariables, IEnumerable<int> rowIndices) {
+      double[] targetVector ;
+      var nRows = rowIndices.Count();
+      if (string.IsNullOrEmpty(targetVariable)) {
+        // if the target variable is not set (e.g. for prediction of a trained model) we just use a zero vector
+        targetVector = new double[nRows];
+      } else {
+        targetVector = dataset.GetDoubleValues(targetVariable, rowIndices).ToArray();
+      }
+      svm_node[][] nodes = new svm_node[nRows][];
+      int maxNodeIndex = 0;
+      int svmProblemRowIndex = 0;
+      List<string> inputVariablesList = inputVariables.ToList();
+      foreach (int row in rowIndices) {
+        List<svm_node> tempRow = new List<svm_node>();
+        int colIndex = 1; // make sure the smallest node index for SVM = 1
+        foreach (var inputVariable in inputVariablesList) {
+          double value = dataset.GetDoubleValue(inputVariable, row);
+          // SVM also works with missing values
+          // => don't add NaN values in the dataset to the sparse SVM matrix representation
+          if (!double.IsNaN(value)) {
+            tempRow.Add(new svm_node() { index = colIndex, value = value });
+            // nodes must be sorted in ascending ordered by column index
+            if (colIndex > maxNodeIndex) maxNodeIndex = colIndex;
+          }
+          colIndex++;
+        }
+        nodes[svmProblemRowIndex++] = tempRow.ToArray();
+      }
+      return new svm_problem { l = targetVector.Length, y = targetVector, x = nodes };
+    }
+
+    /// <summary>
+    /// Transforms <paramref name="dataset"/> into a data structure as needed by libSVM for prediction.
+    /// </summary>
+    /// <param name="dataset">The problem data to transform</param>
+    /// <param name="inputVariables">The selected input variables to include in the svm_problem.</param>
+    /// <param name="rowIndices">The rows of the dataset that should be contained in the resulting SVM-problem</param>
+    /// <returns>A problem data type that can be used for prediction with a trained support vector machine.</returns>
+    public static svm_problem CreateSvmProblem(IDataset dataset, IEnumerable<string> inputVariables, IEnumerable<int> rowIndices) {
+      // for prediction we don't need a target variable
+      return CreateSvmProblem(dataset, string.Empty, inputVariables, rowIndices);
+    }
+
+    /// <summary>
+    /// Instantiate and return a svm_parameter object with default values.
+    /// </summary>
+    /// <returns>A svm_parameter object with default values</returns>
+    public static svm_parameter DefaultParameters() {
+      svm_parameter parameter = new svm_parameter();
+      parameter.svm_type = svm_parameter.NU_SVR;
+      parameter.kernel_type = svm_parameter.RBF;
+      parameter.C = 1;
+      parameter.nu = 0.5;
+      parameter.gamma = 1;
+      parameter.p = 1;
+      parameter.cache_size = 500;
+      parameter.probability = 0;
+      parameter.eps = 0.001;
+      parameter.degree = 3;
+      parameter.shrinking = 1;
+      parameter.coef0 = 0;
+
+      return parameter;
+    }
+
+    public static double CrossValidate(IDataAnalysisProblemData problemData, svm_parameter parameters, int numberOfFolds, bool shuffleFolds = true) {
+      var partitions = GenerateSvmPartitions(problemData, numberOfFolds, shuffleFolds);
+      return CalculateCrossValidationPartitions(partitions, parameters);
+    }
+
+    public static svm_parameter GridSearch(out double cvMse, IDataAnalysisProblemData problemData, Dictionary<string, IEnumerable<double>> parameterRanges, int numberOfFolds, bool shuffleFolds = true, int maxDegreeOfParallelism = 1) {
+      DoubleValue mse = new DoubleValue(Double.MaxValue);
+      var bestParam = DefaultParameters();
+      var crossProduct = parameterRanges.Values.CartesianProduct();
+      var setters = parameterRanges.Keys.Select(GenerateSetter).ToList();
+      var partitions = GenerateSvmPartitions(problemData, numberOfFolds, shuffleFolds);
+
+      var locker = new object(); // for thread synchronization
+      Parallel.ForEach(crossProduct, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
+      parameterCombination => {
+        var parameters = DefaultParameters();
+        var parameterValues = parameterCombination.ToList();
+        for (int i = 0; i < parameterValues.Count; ++i)
+          setters[i](parameters, parameterValues[i]);
+
+        double testMse = CalculateCrossValidationPartitions(partitions, parameters);
+        if (!double.IsNaN(testMse)) {
+          lock (locker) {
+            if (testMse < mse.Value) {
+              mse.Value = testMse;
+              bestParam = (svm_parameter)parameters.Clone();
+            }
+          }
+        }
+      });
+      cvMse = mse.Value;
+      return bestParam;
+    }
+
+    private static double CalculateCrossValidationPartitions(Tuple<svm_problem, svm_problem>[] partitions, svm_parameter parameters) {
+      double avgTestMse = 0;
+      var calc = new OnlineMeanSquaredErrorCalculator();
+      foreach (Tuple<svm_problem, svm_problem> tuple in partitions) {
+        var trainingSvmProblem = tuple.Item1;
+        var testSvmProblem = tuple.Item2;
+        var model = svm.svm_train(trainingSvmProblem, parameters);
+        calc.Reset();
+        for (int i = 0; i < testSvmProblem.l; ++i)
+          calc.Add(testSvmProblem.y[i], svm.svm_predict(model, testSvmProblem.x[i]));
+        double mse = calc.ErrorState == OnlineCalculatorError.None ? calc.MeanSquaredError : double.NaN;
+        avgTestMse += mse;
+      }
+      avgTestMse /= partitions.Length;
+      return avgTestMse;
+    }
+
+    private static Tuple<svm_problem, svm_problem>[] GenerateSvmPartitions(IDataAnalysisProblemData problemData, int numberOfFolds, bool shuffleFolds = true) {
+      var folds = GenerateFolds(problemData, numberOfFolds, shuffleFolds).ToList();
+      var targetVariable = GetTargetVariableName(problemData);
+      var partitions = new Tuple<svm_problem, svm_problem>[numberOfFolds];
+      for (int i = 0; i < numberOfFolds; ++i) {
+        int p = i; // avoid "access to modified closure" warning below
+        var trainingRows = folds.SelectMany((par, j) => j != p ? par : Enumerable.Empty<int>());
+        var testRows = folds[i];
+        var trainingSvmProblem = CreateSvmProblem(problemData.Dataset, targetVariable, problemData.AllowedInputVariables, trainingRows);
+        var rangeTransform = RangeTransform.Compute(trainingSvmProblem);
+        var testSvmProblem = rangeTransform.Scale(CreateSvmProblem(problemData.Dataset, targetVariable, problemData.AllowedInputVariables, testRows));
+        partitions[i] = new Tuple<svm_problem, svm_problem>(rangeTransform.Scale(trainingSvmProblem), testSvmProblem);
+      }
+      return partitions;
+    }
+
+    public static IEnumerable<IEnumerable<int>> GenerateFolds(IDataAnalysisProblemData problemData, int numberOfFolds, bool shuffleFolds = true) {
+      var random = new MersenneTwister((uint)Environment.TickCount);
+      if (problemData is IRegressionProblemData) {
+        var trainingIndices = shuffleFolds ? problemData.TrainingIndices.OrderBy(x => random.Next()) : problemData.TrainingIndices;
+        return GenerateFolds(trainingIndices, problemData.TrainingPartition.Size, numberOfFolds);
+      }
+      if (problemData is IClassificationProblemData) {
+        // when shuffle is enabled do stratified folds generation, some folds may have zero elements 
+        // otherwise, generate folds normally
+        return shuffleFolds ? GenerateFoldsStratified(problemData as IClassificationProblemData, numberOfFolds, random) : GenerateFolds(problemData.TrainingIndices, problemData.TrainingPartition.Size, numberOfFolds);
+      }
+      throw new ArgumentException("Problem data is neither regression or classification problem data.");
+    }
+
+    /// <summary>
+    /// Stratified fold generation from classification data. Stratification means that we ensure the same distribution of class labels for each fold.
+    /// The samples are grouped by class label and each group is split into @numberOfFolds parts. The final folds are formed from the joining of 
+    /// the corresponding parts from each class label.
+    /// </summary>
+    /// <param name="problemData">The classification problem data.</param>
+    /// <param name="numberOfFolds">The number of folds in which to split the data.</param>
+    /// <param name="random">The random generator used to shuffle the folds.</param>
+    /// <returns>An enumerable sequece of folds, where a fold is represented by a sequence of row indices.</returns>
+    private static IEnumerable<IEnumerable<int>> GenerateFoldsStratified(IClassificationProblemData problemData, int numberOfFolds, IRandom random) {
+      var values = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+      var valuesIndices = problemData.TrainingIndices.Zip(values, (i, v) => new { Index = i, Value = v }).ToList();
+      IEnumerable<IEnumerable<IEnumerable<int>>> foldsByClass = valuesIndices.GroupBy(x => x.Value, x => x.Index).Select(g => GenerateFolds(g, g.Count(), numberOfFolds));
+      var enumerators = foldsByClass.Select(f => f.GetEnumerator()).ToList();
+      while (enumerators.All(e => e.MoveNext())) {
+        yield return enumerators.SelectMany(e => e.Current).OrderBy(x => random.Next()).ToList();
+      }
+    }
+
+    private static IEnumerable<IEnumerable<T>> GenerateFolds<T>(IEnumerable<T> values, int valuesCount, int numberOfFolds) {
+      // if number of folds is greater than the number of values, some empty folds will be returned
+      if (valuesCount < numberOfFolds) {
+        for (int i = 0; i < numberOfFolds; ++i)
+          yield return i < valuesCount ? values.Skip(i).Take(1) : Enumerable.Empty<T>();
+      } else {
+        int f = valuesCount / numberOfFolds, r = valuesCount % numberOfFolds; // number of folds rounded to integer and remainder
+        int start = 0, end = f;
+        for (int i = 0; i < numberOfFolds; ++i) {
+          if (r > 0) {
+            ++end;
+            --r;
+          }
+          yield return values.Skip(start).Take(end - start);
+          start = end;
+          end += f;
+        }
+      }
+    }
+
+    private static Action<svm_parameter, double> GenerateSetter(string fieldName) {
+      var targetExp = Expression.Parameter(typeof(svm_parameter));
+      var valueExp = Expression.Parameter(typeof(double));
+      var fieldExp = Expression.Field(targetExp, fieldName);
+      var assignExp = Expression.Assign(fieldExp, Expression.Convert(valueExp, fieldExp.Type));
+      var setter = Expression.Lambda<Action<svm_parameter, double>>(assignExp, targetExp, valueExp).Compile();
+      return setter;
+    }
+
+    private static string GetTargetVariableName(IDataAnalysisProblemData problemData) {
+      var regressionProblemData = problemData as IRegressionProblemData;
+      var classificationProblemData = problemData as IClassificationProblemData;
+
+      if (regressionProblemData != null)
+        return regressionProblemData.TargetVariable;
+      if (classificationProblemData != null)
+        return classificationProblemData.TargetVariable;
+
+      throw new ArgumentException("Problem data is neither regression or classification problem data.");
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegression.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegression.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegression.cs	(revision 17041)
@@ -0,0 +1,271 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+using LibSVM;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Support vector machine regression data analysis algorithm.
+  /// </summary>
+  [Item("Support Vector Regression (SVM)", "Support vector machine regression data analysis algorithm (wrapper for libSVM).")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 110)]
+  [StorableType("645A21E5-EF07-46BF-AA04-A616165F0EF4")]
+  public sealed class SupportVectorRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+    private const string SvmTypeParameterName = "SvmType";
+    private const string KernelTypeParameterName = "KernelType";
+    private const string CostParameterName = "Cost";
+    private const string NuParameterName = "Nu";
+    private const string GammaParameterName = "Gamma";
+    private const string EpsilonParameterName = "Epsilon";
+    private const string DegreeParameterName = "Degree";
+    private const string CreateSolutionParameterName = "CreateSolution";
+
+    #region parameter properties
+    public IConstrainedValueParameter<StringValue> SvmTypeParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[SvmTypeParameterName]; }
+    }
+    public IConstrainedValueParameter<StringValue> KernelTypeParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[KernelTypeParameterName]; }
+    }
+    public IValueParameter<DoubleValue> NuParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters[NuParameterName]; }
+    }
+    public IValueParameter<DoubleValue> CostParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters[CostParameterName]; }
+    }
+    public IValueParameter<DoubleValue> GammaParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters[GammaParameterName]; }
+    }
+    public IValueParameter<DoubleValue> EpsilonParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters[EpsilonParameterName]; }
+    }
+    public IValueParameter<IntValue> DegreeParameter {
+      get { return (IValueParameter<IntValue>)Parameters[DegreeParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> CreateSolutionParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
+    }
+    #endregion
+    #region properties
+    public StringValue SvmType {
+      get { return SvmTypeParameter.Value; }
+      set { SvmTypeParameter.Value = value; }
+    }
+    public StringValue KernelType {
+      get { return KernelTypeParameter.Value; }
+      set { KernelTypeParameter.Value = value; }
+    }
+    public DoubleValue Nu {
+      get { return NuParameter.Value; }
+    }
+    public DoubleValue Cost {
+      get { return CostParameter.Value; }
+    }
+    public DoubleValue Gamma {
+      get { return GammaParameter.Value; }
+    }
+    public DoubleValue Epsilon {
+      get { return EpsilonParameter.Value; }
+    }
+    public IntValue Degree {
+      get { return DegreeParameter.Value; }
+    }
+    public bool CreateSolution {
+      get { return CreateSolutionParameter.Value.Value; }
+      set { CreateSolutionParameter.Value.Value = value; }
+    }
+    #endregion
+    [StorableConstructor]
+    private SupportVectorRegression(StorableConstructorFlag _) : base(_) { }
+    private SupportVectorRegression(SupportVectorRegression original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public SupportVectorRegression()
+      : base() {
+      Problem = new RegressionProblem();
+
+      List<StringValue> svrTypes = (from type in new List<string> { "NU_SVR", "EPSILON_SVR" }
+                                    select new StringValue(type).AsReadOnly())
+                                   .ToList();
+      ItemSet<StringValue> svrTypeSet = new ItemSet<StringValue>(svrTypes);
+      List<StringValue> kernelTypes = (from type in new List<string> { "LINEAR", "POLY", "SIGMOID", "RBF" }
+                                       select new StringValue(type).AsReadOnly())
+                                   .ToList();
+      ItemSet<StringValue> kernelTypeSet = new ItemSet<StringValue>(kernelTypes);
+      Parameters.Add(new ConstrainedValueParameter<StringValue>(SvmTypeParameterName, "The type of SVM to use.", svrTypeSet, svrTypes[0]));
+      Parameters.Add(new ConstrainedValueParameter<StringValue>(KernelTypeParameterName, "The kernel type to use for the SVM.", kernelTypeSet, kernelTypes[3]));
+      Parameters.Add(new ValueParameter<DoubleValue>(NuParameterName, "The value of the nu parameter of the nu-SVR.", new DoubleValue(0.5)));
+      Parameters.Add(new ValueParameter<DoubleValue>(CostParameterName, "The value of the C (cost) parameter of epsilon-SVR and nu-SVR.", new DoubleValue(1.0)));
+      Parameters.Add(new ValueParameter<DoubleValue>(GammaParameterName, "The value of the gamma parameter in the kernel function.", new DoubleValue(1.0)));
+      Parameters.Add(new ValueParameter<DoubleValue>(EpsilonParameterName, "The value of the epsilon parameter for epsilon-SVR.", new DoubleValue(0.1)));
+      Parameters.Add(new ValueParameter<IntValue>(DegreeParameterName, "The degree parameter for the polynomial kernel function.", new IntValue(3)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+      Parameters[CreateSolutionParameterName].Hidden = true;
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      #region backwards compatibility (change with 3.4)
+
+      if (!Parameters.ContainsKey(DegreeParameterName)) {
+        Parameters.Add(new ValueParameter<IntValue>(DegreeParameterName,
+          "The degree parameter for the polynomial kernel function.", new IntValue(3)));
+      }
+      if (!Parameters.ContainsKey(CreateSolutionParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+        Parameters[CreateSolutionParameterName].Hidden = true;
+      }
+      #endregion
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new SupportVectorRegression(this, cloner);
+    }
+
+    #region support vector regression
+    protected override void Run(CancellationToken cancellationToken) {
+      IRegressionProblemData problemData = Problem.ProblemData;
+      IEnumerable<string> selectedInputVariables = problemData.AllowedInputVariables;
+      int nSv;
+      ISupportVectorMachineModel model;
+      Run(problemData, selectedInputVariables, SvmType.Value, KernelType.Value, Cost.Value, Nu.Value, Gamma.Value, Epsilon.Value, Degree.Value, out model, out nSv);
+
+      if (CreateSolution) {
+        var solution = new SupportVectorRegressionSolution((SupportVectorMachineModel)model, (IRegressionProblemData)problemData.Clone());
+        Results.Add(new Result("Support vector regression solution", "The support vector regression solution.", solution));
+      }
+
+      Results.Add(new Result("Number of support vectors", "The number of support vectors of the SVR solution.", new IntValue(nSv)));
+
+
+      {
+        // calculate regression model metrics 
+        var ds = problemData.Dataset;
+        var trainRows = problemData.TrainingIndices;
+        var testRows = problemData.TestIndices;
+        var yTrain = ds.GetDoubleValues(problemData.TargetVariable, trainRows);
+        var yTest = ds.GetDoubleValues(problemData.TargetVariable, testRows);
+        var yPredTrain = model.GetEstimatedValues(ds, trainRows).ToArray();
+        var yPredTest = model.GetEstimatedValues(ds, testRows).ToArray();
+
+        OnlineCalculatorError error;
+        var trainMse = OnlineMeanSquaredErrorCalculator.Calculate(yPredTrain, yTrain, out error);
+        if (error != OnlineCalculatorError.None) trainMse = double.MaxValue;
+        var testMse = OnlineMeanSquaredErrorCalculator.Calculate(yPredTest, yTest, out error);
+        if (error != OnlineCalculatorError.None) testMse = double.MaxValue;
+
+        Results.Add(new Result("Mean squared error (training)", "The mean of squared errors of the SVR solution on the training partition.", new DoubleValue(trainMse)));
+        Results.Add(new Result("Mean squared error (test)", "The mean of squared errors of the SVR solution on the test partition.", new DoubleValue(testMse)));
+
+
+        var trainMae = OnlineMeanAbsoluteErrorCalculator.Calculate(yPredTrain, yTrain, out error);
+        if (error != OnlineCalculatorError.None) trainMae = double.MaxValue;
+        var testMae = OnlineMeanAbsoluteErrorCalculator.Calculate(yPredTest, yTest, out error);
+        if (error != OnlineCalculatorError.None) testMae = double.MaxValue;
+
+        Results.Add(new Result("Mean absolute error (training)", "The mean of absolute errors of the SVR solution on the training partition.", new DoubleValue(trainMae)));
+        Results.Add(new Result("Mean absolute error (test)", "The mean of absolute errors of the SVR solution on the test partition.", new DoubleValue(testMae)));
+
+
+        var trainRelErr = OnlineMeanAbsolutePercentageErrorCalculator.Calculate(yPredTrain, yTrain, out error);
+        if (error != OnlineCalculatorError.None) trainRelErr = double.MaxValue;
+        var testRelErr = OnlineMeanAbsolutePercentageErrorCalculator.Calculate(yPredTest, yTest, out error);
+        if (error != OnlineCalculatorError.None) testRelErr = double.MaxValue;
+
+        Results.Add(new Result("Average relative error (training)", "The mean of relative errors of the SVR solution on the training partition.", new DoubleValue(trainRelErr)));
+        Results.Add(new Result("Average relative error (test)", "The mean of relative errors of the SVR solution on the test partition.", new DoubleValue(testRelErr)));
+      }
+    }
+
+    // BackwardsCompatibility3.4
+    #region Backwards compatible code, remove with 3.5
+    // for compatibility with old API
+    public static SupportVectorRegressionSolution CreateSupportVectorRegressionSolution(
+      IRegressionProblemData problemData, IEnumerable<string> allowedInputVariables,
+      string svmType, string kernelType, double cost, double nu, double gamma, double epsilon, int degree,
+      out double trainingR2, out double testR2, out int nSv) {
+      ISupportVectorMachineModel model;
+      Run(problemData, allowedInputVariables, svmType, kernelType, cost, nu, gamma, epsilon, degree, out model, out nSv);
+
+      var solution = new SupportVectorRegressionSolution((SupportVectorMachineModel)model, (IRegressionProblemData)problemData.Clone());
+      trainingR2 = solution.TrainingRSquared;
+      testR2 = solution.TestRSquared;
+      return solution;
+    }
+    #endregion
+
+    public static void Run(IRegressionProblemData problemData, IEnumerable<string> allowedInputVariables,
+      string svmType, string kernelType, double cost, double nu, double gamma, double epsilon, int degree,
+      out ISupportVectorMachineModel model, out int nSv) {
+      var dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+
+      svm_parameter parameter = new svm_parameter {
+        svm_type = GetSvmType(svmType),
+        kernel_type = GetKernelType(kernelType),
+        C = cost,
+        nu = nu,
+        gamma = gamma,
+        p = epsilon,
+        cache_size = 500,
+        probability = 0,
+        eps = 0.001,
+        degree = degree,
+        shrinking = 1,
+        coef0 = 0
+      };
+
+      svm_problem problem = SupportVectorMachineUtil.CreateSvmProblem(dataset, targetVariable, allowedInputVariables, rows);
+      RangeTransform rangeTransform = RangeTransform.Compute(problem);
+      svm_problem scaledProblem = rangeTransform.Scale(problem);
+      var svmModel = svm.svm_train(scaledProblem, parameter);
+      nSv = svmModel.SV.Length;
+
+      model = new SupportVectorMachineModel(svmModel, rangeTransform, targetVariable, allowedInputVariables);
+    }
+
+    private static int GetSvmType(string svmType) {
+      if (svmType == "NU_SVR") return svm_parameter.NU_SVR;
+      if (svmType == "EPSILON_SVR") return svm_parameter.EPSILON_SVR;
+      throw new ArgumentException("Unknown SVM type");
+    }
+
+    private static int GetKernelType(string kernelType) {
+      if (kernelType == "LINEAR") return svm_parameter.LINEAR;
+      if (kernelType == "POLY") return svm_parameter.POLY;
+      if (kernelType == "SIGMOID") return svm_parameter.SIGMOID;
+      if (kernelType == "RBF") return svm_parameter.RBF;
+      throw new ArgumentException("Unknown kernel type");
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegressionSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegressionSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegressionSolution.cs	(revision 17041)
@@ -0,0 +1,53 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a support vector solution for a regression problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("SupportVectorRegressionSolution", "Represents a support vector solution for a regression problem which can be visualized in the GUI.")]
+  [StorableType("B23D2178-B9B5-4553-A633-DE359AB69BE9")]
+  public sealed class SupportVectorRegressionSolution : RegressionSolution, ISupportVectorMachineSolution {
+
+    public new ISupportVectorMachineModel Model {
+      get { return (ISupportVectorMachineModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private SupportVectorRegressionSolution(StorableConstructorFlag _) : base(_) { }
+    private SupportVectorRegressionSolution(SupportVectorRegressionSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public SupportVectorRegressionSolution(SupportVectorMachineModel model, IRegressionProblemData problemData)
+      : base(model, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new SupportVectorRegressionSolution(this, cloner);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/CosineDistance.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/CosineDistance.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/CosineDistance.cs	(revision 17041)
@@ -0,0 +1,69 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// The angular distance as defined as a normalized distance measure dependent on the angle between two vectors.
+  /// </summary>
+  [StorableType("C87DE522-CB6D-485B-B2F7-6FE79B4E4DC6")]
+  [Item("CosineDistance", "The angular distance as defined as a normalized distance measure dependent on the angle between two vectors.")]
+  public class CosineDistance : DistanceBase<IEnumerable<double>> {
+    #region HLConstructors & Cloning
+    [StorableConstructor]
+    protected CosineDistance(StorableConstructorFlag _) : base(_) { }
+    protected CosineDistance(CosineDistance original, Cloner cloner)
+      : base(original, cloner) { }
+    public CosineDistance() { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CosineDistance(this, cloner);
+    }
+    #endregion
+
+    #region statics
+    public static double GetDistance(IEnumerable<double> point1, IEnumerable<double> point2) {
+      using (IEnumerator<double> p1Enum = point1.GetEnumerator(), p2Enum = point2.GetEnumerator()) {
+        var innerprod = 0.0;
+        var length1 = 0.0;
+        var length2 = 0.0;
+        while (p1Enum.MoveNext() & p2Enum.MoveNext()) {
+          double d1 = p1Enum.Current, d2 = p2Enum.Current;
+          innerprod += d1 * d2;
+          length1 += d1 * d1;
+          length2 += d2 * d2;
+        }
+        var divisor = Math.Sqrt(length1 * length2);
+        if (divisor.IsAlmost(0)) throw new ArgumentException("Cosine distance is not defined on vectors of length 0");
+        if (p1Enum.MoveNext() || p2Enum.MoveNext()) throw new ArgumentException("Cosine distance not defined on vectors of different length");
+        return 1 - innerprod / divisor;
+      }
+    }
+    #endregion
+    public override double Get(IEnumerable<double> a, IEnumerable<double> b) {
+      return GetDistance(a, b);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/DistanceBase.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/DistanceBase.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/DistanceBase.cs	(revision 17041)
@@ -0,0 +1,77 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("58D49402-2AB1-49CF-9AEF-1C599C4B71F7")]
+  public abstract class DistanceBase<T> : Item, IDistance<T> {
+    #region HLConstructors & Cloning
+    [StorableConstructor]
+    protected DistanceBase(StorableConstructorFlag _) : base(_) { }
+    protected DistanceBase(DistanceBase<T> original, Cloner cloner) : base(original, cloner) { }
+    protected DistanceBase() { }
+    #endregion
+
+    public abstract double Get(T a, T b);
+
+    public IComparer<T> GetDistanceComparer(T item) {
+      return new DistanceComparer(item, this);
+    }
+
+    public double Get(object x, object y) {
+      return Get((T)x, (T)y);
+    }
+
+    public IComparer GetDistanceComparer(object item) {
+      return new DistanceComparer((T)item, this);
+    }
+
+    [StorableType("35fcecb5-9209-469b-9e4e-82efd8abfddf")]
+    internal class DistanceComparer : IComparer<T>, IComparer {
+      [Storable]
+      private readonly T item;
+      [Storable]
+      private readonly IDistance<T> dist;
+
+      [StorableConstructor]
+      protected DistanceComparer(StorableConstructorFlag _) {
+      }
+
+      public DistanceComparer(T item, IDistance<T> dist) {
+        this.dist = dist;
+        this.item = item;
+      }
+
+      public int Compare(T x, T y) {
+        return dist.Get(x, item).CompareTo(dist.Get(y, item));
+      }
+
+      public int Compare(object x, object y) {
+        return Compare((T)x, (T)y);
+      }
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/EuclideanDistance.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/EuclideanDistance.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/EuclideanDistance.cs	(revision 17041)
@@ -0,0 +1,58 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("1D3FE1D5-E524-4DEC-B845-34C940F5BA61")]
+  [Item("EuclideanDistance", "A norm function that uses Euclidean distance")]
+  public class EuclideanDistance : DistanceBase<IEnumerable<double>> {
+    #region HLConstructors & Cloning
+    [StorableConstructor]
+    protected EuclideanDistance(StorableConstructorFlag _) : base(_) { }
+    protected EuclideanDistance(EuclideanDistance original, Cloner cloner) : base(original, cloner) { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new EuclideanDistance(this, cloner);
+    }
+    public EuclideanDistance() { }
+    #endregion
+
+    public static double GetDistance(IEnumerable<double> point1, IEnumerable<double> point2) {
+      using (IEnumerator<double> p1Enum = point1.GetEnumerator(), p2Enum = point2.GetEnumerator()) {
+        var sum = 0.0;
+        while (p1Enum.MoveNext() & p2Enum.MoveNext()) {
+          var d = p1Enum.Current - p2Enum.Current;
+          sum += d * d;
+        }
+        if (p1Enum.MoveNext() || p2Enum.MoveNext()) throw new ArgumentException("Euclidean distance not defined on vectors of different length");
+        return Math.Sqrt(sum);
+      }
+    }
+
+    public override double Get(IEnumerable<double> a, IEnumerable<double> b) {
+      return GetDistance(a, b);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/IDistance.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/IDistance.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/IDistance.cs	(revision 17041)
@@ -0,0 +1,52 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections;
+using System.Collections.Generic;
+using HEAL.Attic;
+using HeuristicLab.Core;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("168066b5-a14b-4d87-b3fb-6bac363e9c98")]
+  public interface IDistance<in T> : IDistance {
+    /// <summary>
+    /// Calculates a distance measure between two objects.
+    /// 1.) non-negative d(x,y) >= 0
+    /// 2.) symmetric d(x,y) = d(y,x)
+    /// 3.) zero-reflexive d(x,x) = 0;
+    /// </summary>
+    /// <returns>d(x,y)</returns>
+    double Get(T x, T y);
+
+    /// <summary>
+    /// Returns a comparator wich compares the distances to item. (allows for sorting nearest/farthest neighbours)
+    /// </summary>
+    /// <param name="item"></param>
+    /// <returns></returns>
+    IComparer<T> GetDistanceComparer(T item);
+  }
+
+  [StorableType("3e44c812-79ad-404d-95e9-8d9c3467110c")]
+  public interface IDistance : IItem {
+    double Get(object x, object y);
+    IComparer GetDistanceComparer(object item);
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/IndexedItemDistance.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/IndexedItemDistance.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/IndexedItemDistance.cs	(revision 17041)
@@ -0,0 +1,54 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Collections;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("D6D34B11-7618-4F8E-AFC3-3E42DAA1E38C")]
+  [Item("IndexedItemDistance", "A distance wrapper for indexed items")]
+  internal class IndexedItemDistance<T> : DistanceBase<IndexedItem<T>> {
+    [Storable]
+    private readonly IDistance<T> dist;
+
+    #region HLConstructors & Cloning
+
+    [StorableConstructor]
+    protected IndexedItemDistance(StorableConstructorFlag _) : base(_) {
+    }
+
+    protected IndexedItemDistance(IndexedItemDistance<T> original, Cloner cloner) : base(original, cloner) {
+      dist = cloner.Clone(original.dist);
+    }
+    public override IDeepCloneable Clone(Cloner cloner) { return new IndexedItemDistance<T>(this, cloner); }
+    public IndexedItemDistance(IDistance<T> distance) {
+      dist = distance;
+    }
+    #endregion
+
+    public override double Get(IndexedItem<T> a, IndexedItem<T> b) {
+      return dist.Get(a.Value, b.Value);
+    }
+  }
+}
+
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/ManhattanDistance.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/ManhattanDistance.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/ManhattanDistance.cs	(revision 17041)
@@ -0,0 +1,59 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("65C04216-1441-41EE-91B7-A80B2AD5E332")]
+  [Item("ManhattanDistance", "A distance function that uses block distance")]
+  public class ManhattanDistance : DistanceBase<IEnumerable<double>> {
+    #region HLConstructors & Cloning
+    [StorableConstructor]
+    protected ManhattanDistance(StorableConstructorFlag _) : base(_) { }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+    protected ManhattanDistance(ManhattanDistance original, Cloner cloner)
+      : base(original, cloner) { }
+    public ManhattanDistance() { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new ManhattanDistance(this, cloner);
+    }
+    #endregion
+
+    public static double GetDistance(IEnumerable<double> point1, IEnumerable<double> point2) {
+      using (IEnumerator<double> p1Enum = point1.GetEnumerator(), p2Enum = point2.GetEnumerator()) {
+        var sum = 0.0;
+        while (p1Enum.MoveNext() & p2Enum.MoveNext())
+          sum += Math.Abs(p1Enum.Current - p2Enum.Current);
+        if (p1Enum.MoveNext() || p2Enum.MoveNext()) throw new ArgumentException("Manhattan distance not defined on vectors of different length");
+        return sum;
+      }
+    }
+
+    public override double Get(IEnumerable<double> a, IEnumerable<double> b) {
+      return GetDistance(a, b);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/WeightedEuclideanDistance.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/WeightedEuclideanDistance.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/WeightedEuclideanDistance.cs	(revision 17041)
@@ -0,0 +1,126 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("A660B39D-1B11-459D-98C2-65942E6D375C")]
+  [Item("WeightedEuclideanDistance", "A weighted norm function that uses Euclidean distance √(Σ(w[i]²*(p1[i]-p2[i])²))")]
+  public class WeightedEuclideanDistance : ParameterizedNamedItem, IDistance<IEnumerable<double>> {
+    [Storable]
+    private double[] weights;
+    public const string WeightsParameterName = "Weights";
+    public IValueParameter<DoubleArray> WeightsParameter {
+      get { return (IValueParameter<DoubleArray>) Parameters[WeightsParameterName]; }
+    }
+
+    public DoubleArray Weights {
+      get { return WeightsParameter.Value; }
+      set { WeightsParameter.Value = value; }
+    }
+
+    #region HLConstructors & Cloning
+    [StorableConstructor]
+    protected WeightedEuclideanDistance(StorableConstructorFlag _) : base(_) { }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      RegisterParameterEvents();
+    }
+    protected WeightedEuclideanDistance(WeightedEuclideanDistance original, Cloner cloner) : base(original, cloner) {
+      RegisterParameterEvents();
+      weights = original.weights != null ? original.weights.ToArray() : null;
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new WeightedEuclideanDistance(this, cloner);
+    }
+    public WeightedEuclideanDistance() {
+      Parameters.Add(new ValueParameter<DoubleArray>(WeightsParameterName, "The weights used to modify the euclidean distance.", new DoubleArray(new[] {1.0})));
+      RegisterParameterEvents();
+    }
+    #endregion
+
+    public static double GetDistance(IEnumerable<double> point1, IEnumerable<double> point2, IEnumerable<double> weights) {
+      using (IEnumerator<double> p1Enum = point1.GetEnumerator(), p2Enum = point2.GetEnumerator(), weEnum = weights.GetEnumerator()) {
+        var sum = 0.0;
+        while (p1Enum.MoveNext() & p2Enum.MoveNext() & weEnum.MoveNext()) {
+          var d = p1Enum.Current - p2Enum.Current;
+          var w = weEnum.Current;
+          sum += d * d * w * w;
+        }
+        if (weEnum.MoveNext() || p1Enum.MoveNext() || p2Enum.MoveNext()) throw new ArgumentException("Weighted Euclidean distance not defined on vectors of different length");
+        return Math.Sqrt(sum);
+      }
+    }
+
+    public double Get(IEnumerable<double> a, IEnumerable<double> b) {
+      return GetDistance(a, b, weights);
+    }
+    public IComparer<IEnumerable<double>> GetDistanceComparer(IEnumerable<double> item) {
+      return new DistanceBase<IEnumerable<double>>.DistanceComparer(item, this);
+    }
+    public double Get(object x, object y) {
+      return Get((IEnumerable<double>) x, (IEnumerable<double>) y);
+    }
+    public IComparer GetDistanceComparer(object item) {
+      return new DistanceBase<IEnumerable<double>>.DistanceComparer((IEnumerable<double>) item, this);
+    }
+
+    public void AdaptToProblemData(IDataAnalysisProblemData problemData) {
+      Weights = new DoubleArray(problemData.AllowedInputVariables.Select(v => Weights.ElementNames.Contains(v) ? GetWeight(v) : 1).ToArray())
+        {ElementNames = problemData.AllowedInputVariables};
+    }
+    public void Initialize(IDataAnalysisProblemData problemData) {
+      if (Weights.Length != problemData.AllowedInputVariables.Count()) throw new ArgumentException("Number of Weights does not match the number of input variables");
+      weights = Weights.ElementNames.All(v => v == null || v.Equals(string.Empty)) ? 
+        Weights.ToArray() : 
+        problemData.AllowedInputVariables.Select(GetWeight).ToArray();
+    }
+    private double GetWeight(string v) {
+      var w = Weights;
+      var names = w.ElementNames.ToArray();
+      for (var i = 0; i < w.Length; i++) if (names[i].Equals(v)) return w[i];
+      throw new ArgumentException("weigth for " + v + " was requested but not specified.");
+    }
+    private void RegisterParameterEvents() {
+      WeightsParameter.ValueChanged += OnWeightsArrayChanged;
+      WeightsParameter.Value.ItemChanged += OnWeightChanged;
+    }
+    private void OnWeightChanged(object sender, EventArgs<int> e) {
+      WeightsParameter.Value.ItemChanged -= OnWeightChanged;
+      Weights[e.Value] = Math.Max(0, Weights[e.Value]);
+      WeightsParameter.Value.ItemChanged -= OnWeightChanged;
+    }
+    private void OnWeightsArrayChanged(object sender, EventArgs e) {
+      for (var i = 0; i < Weights.Length; i++)
+        Weights[i] = Math.Max(0, Weights[i]);
+      WeightsParameter.Value.ItemChanged += OnWeightChanged;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/PriorityQueue.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/PriorityQueue.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/PriorityQueue.cs	(revision 17041)
@@ -0,0 +1,190 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+
+  // Code Taken from branch: RoutePlanning, Heap4
+  // implementation based on C++ version from Peter Sanders
+  // http://www.mpi-inf.mpg.de/~sanders/programs/spq/
+  public sealed class PriorityQueue<TK, TV> where TK : IComparable {
+    private class KNElement {
+      public TK Key { get; set; }
+      public TV Value { get; set; }
+    }
+
+    private readonly int capacity;
+    private int size; // index of last used element
+    private int finalLayerSize; // size of first layer with free space
+    private int finalLayerDist; // distance to end of layer
+                                //private KNElement[] rawData;
+    private readonly KNElement[] data;    // aligned version of rawData
+
+    public PriorityQueue(TK supremum, TK infimum, int cap) {
+      capacity = cap;
+      data = Enumerable.Range(0, capacity + 4).Select(x => new KNElement()).ToArray();
+
+      data[0].Key = infimum; // sentinel
+      data[capacity + 1].Key = supremum;
+      data[capacity + 2].Key = supremum;
+      data[capacity + 3].Key = supremum;
+      Reset();
+    }
+
+    public int Size {
+      get { return size; }
+    }
+
+    public TK Supremum {
+      get { return data[capacity + 1].Key; }
+    }
+
+    public KeyValuePair<TK, TV> PeekMin() {
+      if (size == 0) throw new InvalidOperationException("Heap is empty");
+      return new KeyValuePair<TK, TV>(data[1].Key, data[1].Value);
+    }
+
+    public TK PeekMinKey() {
+      return data[1].Key;
+    }
+
+    public TV PeekMinValue() {
+      return data[1].Value;
+    }
+
+    public void RemoveMin() {
+      // first move up elements on a min-path
+      var hole = 1;
+      var succ = 2;
+      var layerSize = 4;
+      var layerPos = 0;
+      var sz = size;
+      size = sz - 1;
+      finalLayerDist++;
+      if (finalLayerDist == finalLayerSize) {
+        finalLayerSize >>= 2;
+        finalLayerDist = 0;
+      }
+
+      while (succ < sz) {
+        var minKey = data[succ].Key;
+        var delta = 0;
+
+        for (var i = 1; i <= 3; i++) {
+          var otherKey = data[succ + i].Key;
+          if (otherKey.CompareTo(minKey) >= 0) continue;
+          minKey = otherKey;
+          delta = i;
+        }
+        
+        succ += delta;
+        layerPos += delta;
+
+        // move min successor up
+        data[hole].Key = minKey;
+        data[hole].Value = data[succ].Value;
+
+        // step to next layer
+        hole = succ;
+        succ = succ - layerPos + layerSize; // beginning of next layer
+        layerPos <<= 2;
+        succ += layerPos; // now correct value
+        layerSize <<= 2;
+      }
+
+      // bubble up rightmost element
+      var bubble = data[sz].Key;
+      layerSize >>= 2; // now size of hole's layer
+      layerPos >>= 2; // now pos of hole within its layer
+
+      var layerDist = layerSize - layerPos - 1; // hole's dist to end of layer
+      var pred = hole + layerDist - layerSize; // end of pred's layer for now
+      layerSize >>= 2; // now size of pred's layer
+      layerDist >>= 2; // now pred's pos in layer
+      pred = pred - layerDist; // finally preds index
+
+      while (data[pred].Key.CompareTo(bubble) > 0) {  // must terminate since inf at root
+        data[hole].Key = data[pred].Key;
+        data[hole].Value = data[pred].Value;
+        hole = pred;
+        pred = hole + layerDist - layerSize; // end of hole's layer for now
+        layerSize >>= 2;
+        layerDist >>= 2;
+        pred = pred - layerDist; // finally preds index
+      }
+
+      // finally move data to hole
+      data[hole].Key = bubble;
+      data[hole].Value = data[sz].Value;
+      data[sz].Key = Supremum; // mark as deleted
+    }
+
+    public void Insert(TK key, TV value) {
+      var layerSize = finalLayerSize;
+      var layerDist = finalLayerDist;
+      finalLayerDist--;
+
+      if (finalLayerDist == -1) { // layer full
+                                  // start next layer
+        finalLayerSize <<= 2;
+        finalLayerDist = finalLayerSize - 1;
+      }
+
+      size++;
+
+      var hole = size;
+      var pred = hole + layerDist - layerSize; // end of pred's layer for now
+      layerSize >>= 2; // now size of pred's layer
+      layerDist >>= 2;
+      pred = pred - layerDist; // finally preds index
+      var predKey = data[pred].Key;
+      while (predKey.CompareTo(key) > 0) {
+        data[hole].Key = predKey;
+        data[hole].Value = data[pred].Value;
+        hole = pred;
+        pred = hole + layerDist - layerSize; // end of pred's layer for now
+        layerSize >>= 2; // now isze of pred's layer
+        layerDist >>= 2;
+        pred = pred - layerDist; // finally preds index
+        predKey = data[pred].Key;
+      }
+
+      // finally move data to hole
+      data[hole].Key = key;
+      data[hole].Value = value;
+    }
+
+    // reset size to 0 and fill data array with sentinels
+    private void Reset() {
+      size = 0;
+      finalLayerSize = 1;
+      finalLayerDist = 0;
+      var sup = Supremum;
+      var cap = capacity;
+      for (var i = 1; i <= cap; i++) {
+        data[i].Key = sup;
+      }
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/SpacePartitioningTree.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/SpacePartitioningTree.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/SpacePartitioningTree.cs	(revision 17041)
@@ -0,0 +1,285 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//Code is based on an implementation from Laurens van der Maaten
+
+/*
+*
+* Copyright (c) 2014, Laurens van der Maaten (Delft University of Technology)
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following conditions are met:
+* 1. Redistributions of source code must retain the above copyright
+*    notice, this list of conditions and the following disclaimer.
+* 2. Redistributions in binary form must reproduce the above copyright
+*    notice, this list of conditions and the following disclaimer in the
+*    documentation and/or other materials provided with the distribution.
+* 3. All advertising materials mentioning features or use of this software
+*    must display the following acknowledgement:
+*    This product includes software developed by the Delft University of Technology.
+* 4. Neither the name of the Delft University of Technology nor the names of
+*    its contributors may be used to endorse or promote products derived from
+*    this software without specific prior written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY LAURENS VAN DER MAATEN ''AS IS'' AND ANY EXPRESS
+* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+* EVENT SHALL LAURENS VAN DER MAATEN BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+* OF SUCH DAMAGE.
+*
+*/
+#endregion
+
+using System;
+using System.Collections.Generic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Space partitioning tree (SPTree)
+  /// </summary>
+  internal class SpacePartitioningTree {
+    private const uint QtNodeCapacity = 1;
+
+    #region Fields
+    private int dimension;
+    private bool isLeaf;
+    private uint size;
+    private uint cumulativeSize;
+
+    // Axis-aligned bounding box stored as a center with half-dimensions to represent the boundaries of this quad tree
+    private Cell boundary;
+
+    private double[,] data;
+
+    // Indices in this space-partitioning tree node, corresponding center-of-mass, and list of all children
+    private double[] centerOfMass;
+    private readonly int[] index = new int[QtNodeCapacity];
+
+    // Children
+    private SpacePartitioningTree[] children;
+    private uint noChildren;
+    #endregion
+
+    public SpacePartitioningTree(double[,] inpData) {
+      var d = inpData.GetLength(1);
+      var n = inpData.GetLength(0);
+      var meanY = new double[d];
+      var minY = new double[d];
+      for (var i = 0; i < d; i++) minY[i] = double.MaxValue;
+      var maxY = new double[d];
+      for (var i = 0; i < d; i++) maxY[i] = double.MinValue;
+      for (uint i = 0; i < n; i++) {
+        for (uint j = 0; j < d; j++) {
+          meanY[j] += inpData[i, j];
+          if (inpData[i, j] < minY[j]) minY[j] = inpData[i, j];
+          if (inpData[i, j] > maxY[j]) maxY[j] = inpData[i, j];
+        }
+      }
+      for (var i = 0; i < d; i++) meanY[i] /= n;
+      var width = new double[d];
+      for (var i = 0; i < d; i++) width[i] = Math.Max(maxY[i] - meanY[i], meanY[i] - minY[i]) + 1e-5;
+      Init(inpData, meanY, width);
+      Fill(n);
+    }
+
+    private SpacePartitioningTree(double[,] inpData, IEnumerable<double> impCorner, IEnumerable<double> impWith) {
+      Init(inpData, impCorner, impWith);
+    }
+
+    public bool Insert(int newIndex) {
+      // Ignore objects which do not belong in this quad tree
+      var point = new double[dimension];
+      Buffer.BlockCopy(data, sizeof(double) * dimension * newIndex, point, 0, sizeof(double) * dimension);
+      if (!boundary.ContainsPoint(point)) return false;
+      cumulativeSize++;
+      // Online update of cumulative size and center-of-mass
+      var mult1 = (double)(cumulativeSize - 1) / cumulativeSize;
+      var mult2 = 1.0 / cumulativeSize;
+      for (var i = 0; i < dimension; i++) centerOfMass[i] *= mult1;
+      for (var i = 0; i < dimension; i++) centerOfMass[i] += mult2 * point[i];
+
+      // If there is space in this quad tree and it is a leaf, add the object here
+      if (isLeaf && size < QtNodeCapacity) {
+        index[size] = newIndex;
+        size++;
+        return true;
+      }
+
+      // Don't add duplicates
+      var anyDuplicate = false;
+      for (uint n = 0; n < size; n++) {
+        var duplicate = true;
+        for (var d = 0; d < dimension; d++) {
+          if (Math.Abs(point[d] - data[index[n], d]) < double.Epsilon) continue;
+          duplicate = false; break;
+        }
+        anyDuplicate = anyDuplicate | duplicate;
+      }
+      if (anyDuplicate) return true;
+
+      // Otherwise, we need to subdivide the current cell
+      if (isLeaf) Subdivide();
+      // Find out where the point can be inserted
+      for (var i = 0; i < noChildren; i++) {
+        if (children[i].Insert(newIndex)) return true;
+      }
+
+      // Otherwise, the point cannot be inserted (this should never happen)
+      return false;
+    }
+
+    public void ComputeNonEdgeForces(int pointIndex, double theta, double[] negF, ref double sumQ) {
+      // Make sure that we spend no time on empty nodes or self-interactions
+      if (cumulativeSize == 0 || (isLeaf && size == 1 && index[0] == pointIndex)) return;
+
+      // Compute distance between point and center-of-mass
+      var D = .0;
+      var buff = new double[dimension];
+      for (var d = 0; d < dimension; d++) buff[d] = data[pointIndex, d] - centerOfMass[d];
+      for (var d = 0; d < dimension; d++) D += buff[d] * buff[d];
+
+      // Check whether we can use this node as a "summary"
+      var maxWidth = 0.0;
+      for (var d = 0; d < dimension; d++) {
+        var curWidth = boundary.GetWidth(d);
+        maxWidth = maxWidth > curWidth ? maxWidth : curWidth;
+      }
+      if (isLeaf || maxWidth / Math.Sqrt(D) < theta) {
+
+        // Compute and add t-SNE force between point and current node
+        D = 1.0 / (1.0 + D);
+        var mult = cumulativeSize * D;
+        sumQ += mult;
+        mult *= D;
+        for (var d = 0; d < dimension; d++) negF[d] += mult * buff[d];
+      } else {
+
+        // Recursively apply Barnes-Hut to children
+        for (var i = 0; i < noChildren; i++) children[i].ComputeNonEdgeForces(pointIndex, theta, negF, ref sumQ);
+      }
+    }
+
+    public static void ComputeEdgeForces(int[] rowP, int[] colP, double[] valP, int n, double[,] posF, double[,] data, int dimension) {
+      // Loop over all edges in the graph
+      for (var k = 0; k < n; k++) {
+        for (var i = rowP[k]; i < rowP[k + 1]; i++) {
+
+          // Compute pairwise distance and Q-value
+          // uses squared distance
+          var d = 1.0;
+          var buff = new double[dimension];
+          for (var j = 0; j < dimension; j++) buff[j] = data[k, j] - data[colP[i], j];
+          for (var j = 0; j < dimension; j++) d += buff[j] * buff[j];
+          d = valP[i] / d;
+
+          // Sum positive force
+          for (var j = 0; j < dimension; j++) posF[k, j] += d * buff[j];
+        }
+      }
+    }
+
+    #region Helpers
+    private void Fill(int n) {
+      for (var i = 0; i < n; i++) Insert(i);
+    }
+
+    private void Init(double[,] inpData, IEnumerable<double> inpCorner, IEnumerable<double> inpWidth) {
+      dimension = inpData.GetLength(1);
+      noChildren = 2;
+      for (uint i = 1; i < dimension; i++) noChildren *= 2;
+      data = inpData;
+      isLeaf = true;
+      size = 0;
+      cumulativeSize = 0;
+      boundary = new Cell((uint)dimension);
+
+      inpCorner.ForEach((i, x) => boundary.SetCorner(i, x));
+      inpWidth.ForEach((i, x) => boundary.SetWidth(i, x));
+
+      children = new SpacePartitioningTree[noChildren];
+      centerOfMass = new double[dimension];
+    }
+
+    private void Subdivide() {
+      // Create new children
+      var newCorner = new double[dimension];
+      var newWidth = new double[dimension];
+      for (var i = 0; i < noChildren; i++) {
+        var div = 1;
+        for (var d = 0; d < dimension; d++) {
+          newWidth[d] = .5 * boundary.GetWidth(d);
+          if (i / div % 2 == 1) newCorner[d] = boundary.GetCorner(d) - .5 * boundary.GetWidth(d);
+          else newCorner[d] = boundary.GetCorner(d) + .5 * boundary.GetWidth(d);
+          div *= 2;
+        }
+        children[i] = new SpacePartitioningTree(data, newCorner, newWidth);
+      }
+
+      // Move existing points to correct children
+      for (var i = 0; i < size; i++) {
+        var success = false;
+        for (var j = 0; j < noChildren; j++) {
+          if (!success) success = children[j].Insert(index[i]);
+        }
+        index[i] = -1; // as in tSNE implementation by van der Maaten
+      }
+      // Empty parent node
+      size = 0;
+      isLeaf = false;
+    }
+    #endregion
+
+    private class Cell {
+      private readonly uint dimension;
+      private readonly double[] corner;
+      private readonly double[] width;
+
+      public Cell(uint inpDimension) {
+        dimension = inpDimension;
+        corner = new double[dimension];
+        width = new double[dimension];
+      }
+
+      public double GetCorner(int d) {
+        return corner[d];
+      }
+      public double GetWidth(int d) {
+        return width[d];
+      }
+      public void SetCorner(int d, double val) {
+        corner[d] = val;
+      }
+      public void SetWidth(int d, double val) {
+        width[d] = val;
+      }
+      public bool ContainsPoint(double[] point) {
+        for (var d = 0; d < dimension; d++)
+          if (corner[d] - width[d] > point[d] || corner[d] + width[d] < point[d]) return false;
+        return true;
+      }
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/TSNEAlgorithm.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/TSNEAlgorithm.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/TSNEAlgorithm.cs	(revision 17041)
@@ -0,0 +1,556 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Analysis;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.PluginInfrastructure;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Random;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// t-Distributed Stochastic Neighbor Embedding (tSNE) projects the data in a low dimensional 
+  /// space to allow visual cluster identification.
+  /// </summary>
+  [Item("t-Distributed Stochastic Neighbor Embedding (tSNE)", "t-Distributed Stochastic Neighbor Embedding projects the data in a low " +
+                                                              "dimensional space to allow visual cluster identification. Implemented similar to: https://lvdmaaten.github.io/tsne/#implementations (Barnes-Hut t-SNE). Described in : https://lvdmaaten.github.io/publications/papers/JMLR_2014.pdf")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 100)]
+  [StorableType("1CE58B5E-C319-4DEB-B66B-994171370B06")]
+  public sealed class TSNEAlgorithm : BasicAlgorithm {
+    public override bool SupportsPause {
+      get { return true; }
+    }
+    public override Type ProblemType {
+      get { return typeof(IDataAnalysisProblem); }
+    }
+    public new IDataAnalysisProblem Problem {
+      get { return (IDataAnalysisProblem)base.Problem; }
+      set { base.Problem = value; }
+    }
+
+    #region Parameter names
+    private const string DistanceFunctionParameterName = "DistanceFunction";
+    private const string PerplexityParameterName = "Perplexity";
+    private const string ThetaParameterName = "Theta";
+    private const string NewDimensionsParameterName = "Dimensions";
+    private const string MaxIterationsParameterName = "MaxIterations";
+    private const string StopLyingIterationParameterName = "StopLyingIteration";
+    private const string MomentumSwitchIterationParameterName = "MomentumSwitchIteration";
+    private const string InitialMomentumParameterName = "InitialMomentum";
+    private const string FinalMomentumParameterName = "FinalMomentum";
+    private const string EtaParameterName = "Eta";
+    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+    private const string SeedParameterName = "Seed";
+    private const string ClassesNameParameterName = "ClassesName";
+    private const string NormalizationParameterName = "Normalization";
+    private const string RandomInitializationParameterName = "RandomInitialization";
+    private const string UpdateIntervalParameterName = "UpdateInterval";
+    #endregion
+
+    #region Result names
+    private const string IterationResultName = "Iteration";
+    private const string ErrorResultName = "Error";
+    private const string ErrorPlotResultName = "Error plot";
+    private const string ScatterPlotResultName = "Scatterplot";
+    private const string DataResultName = "Projected data";
+    #endregion
+
+    #region Parameter properties
+    public IFixedValueParameter<DoubleValue> PerplexityParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[PerplexityParameterName]; }
+    }
+    public IFixedValueParameter<PercentValue> ThetaParameter {
+      get { return (IFixedValueParameter<PercentValue>)Parameters[ThetaParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NewDimensionsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NewDimensionsParameterName]; }
+    }
+    public IConstrainedValueParameter<IDistance<double[]>> DistanceFunctionParameter {
+      get { return (IConstrainedValueParameter<IDistance<double[]>>)Parameters[DistanceFunctionParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> MaxIterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[MaxIterationsParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> StopLyingIterationParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[StopLyingIterationParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> MomentumSwitchIterationParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[MomentumSwitchIterationParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> InitialMomentumParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[InitialMomentumParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> FinalMomentumParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[FinalMomentumParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> EtaParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[EtaParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> SeedParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+    public IConstrainedValueParameter<StringValue> ClassesNameParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[ClassesNameParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> NormalizationParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[NormalizationParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> RandomInitializationParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[RandomInitializationParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> UpdateIntervalParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[UpdateIntervalParameterName]; }
+    }
+    #endregion
+
+    #region  Properties
+    public IDistance<double[]> DistanceFunction {
+      get { return DistanceFunctionParameter.Value; }
+    }
+    public double Perplexity {
+      get { return PerplexityParameter.Value.Value; }
+      set { PerplexityParameter.Value.Value = value; }
+    }
+    public double Theta {
+      get { return ThetaParameter.Value.Value; }
+      set { ThetaParameter.Value.Value = value; }
+    }
+    public int NewDimensions {
+      get { return NewDimensionsParameter.Value.Value; }
+      set { NewDimensionsParameter.Value.Value = value; }
+    }
+    public int MaxIterations {
+      get { return MaxIterationsParameter.Value.Value; }
+      set { MaxIterationsParameter.Value.Value = value; }
+    }
+    public int StopLyingIteration {
+      get { return StopLyingIterationParameter.Value.Value; }
+      set { StopLyingIterationParameter.Value.Value = value; }
+    }
+    public int MomentumSwitchIteration {
+      get { return MomentumSwitchIterationParameter.Value.Value; }
+      set { MomentumSwitchIterationParameter.Value.Value = value; }
+    }
+    public double InitialMomentum {
+      get { return InitialMomentumParameter.Value.Value; }
+      set { InitialMomentumParameter.Value.Value = value; }
+    }
+    public double FinalMomentum {
+      get { return FinalMomentumParameter.Value.Value; }
+      set { FinalMomentumParameter.Value.Value = value; }
+    }
+    public double Eta {
+      get { return EtaParameter.Value.Value; }
+      set { EtaParameter.Value.Value = value; }
+    }
+    public bool SetSeedRandomly {
+      get { return SetSeedRandomlyParameter.Value.Value; }
+      set { SetSeedRandomlyParameter.Value.Value = value; }
+    }
+    public int Seed {
+      get { return SeedParameter.Value.Value; }
+      set { SeedParameter.Value.Value = value; }
+    }
+    public string ClassesName {
+      get { return ClassesNameParameter.Value != null ? ClassesNameParameter.Value.Value : null; }
+      set { ClassesNameParameter.Value.Value = value; }
+    }
+    public bool Normalization {
+      get { return NormalizationParameter.Value.Value; }
+      set { NormalizationParameter.Value.Value = value; }
+    }
+    public bool RandomInitialization {
+      get { return RandomInitializationParameter.Value.Value; }
+      set { RandomInitializationParameter.Value.Value = value; }
+    }
+    public int UpdateInterval {
+      get { return UpdateIntervalParameter.Value.Value; }
+      set { UpdateIntervalParameter.Value.Value = value; }
+    }
+    #endregion
+
+    #region Storable poperties
+    [Storable]
+    private Dictionary<string, IList<int>> dataRowIndices;
+    [Storable]
+    private TSNEStatic<double[]>.TSNEState state;
+    #endregion
+
+    #region Constructors & Cloning
+    [StorableConstructor]
+    private TSNEAlgorithm(StorableConstructorFlag _) : base(_) { }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      if (!Parameters.ContainsKey(RandomInitializationParameterName))
+        Parameters.Add(new FixedValueParameter<BoolValue>(RandomInitializationParameterName, "Wether data points should be randomly initialized or according to the first 2 dimensions", new BoolValue(true)));
+      RegisterParameterEvents();
+    }
+    private TSNEAlgorithm(TSNEAlgorithm original, Cloner cloner) : base(original, cloner) {
+      if (original.dataRowIndices != null)
+        dataRowIndices = new Dictionary<string, IList<int>>(original.dataRowIndices);
+      if (original.state != null)
+        state = cloner.Clone(original.state);
+      RegisterParameterEvents();
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new TSNEAlgorithm(this, cloner);
+    }
+    public TSNEAlgorithm() {
+      var distances = new ItemSet<IDistance<double[]>>(ApplicationManager.Manager.GetInstances<IDistance<double[]>>());
+      Parameters.Add(new ConstrainedValueParameter<IDistance<double[]>>(DistanceFunctionParameterName, "The distance function used to differentiate similar from non-similar points", distances, distances.OfType<EuclideanDistance>().FirstOrDefault()));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(PerplexityParameterName, "Perplexity-parameter of tSNE. Comparable to k in a k-nearest neighbour algorithm. Recommended value is floor(number of points /3) or lower", new DoubleValue(25)));
+      Parameters.Add(new FixedValueParameter<PercentValue>(ThetaParameterName, "Value describing how much appoximated " +
+                                                                               "gradients my differ from exact gradients. Set to 0 for exact calculation and in [0,1] otherwise. " +
+                                                                               "Appropriate values for theta are between 0.1 and 0.7 (default = 0.5). CAUTION: exact calculation of " +
+                                                                               "forces requires building a non-sparse N*N matrix where N is the number of data points. This may " +
+                                                                               "exceed memory limitations. The function is designed to run on large (N > 5000) data sets. It may give" +
+                                                                               " poor performance on very small data sets(it is better to use a standard t - SNE implementation on such data).", new PercentValue(0)));
+      Parameters.Add(new FixedValueParameter<IntValue>(NewDimensionsParameterName, "Dimensionality of projected space (usually 2 for easy visual analysis)", new IntValue(2)));
+      Parameters.Add(new FixedValueParameter<IntValue>(MaxIterationsParameterName, "Maximum number of iterations for gradient descent.", new IntValue(1000)));
+      Parameters.Add(new FixedValueParameter<IntValue>(StopLyingIterationParameterName, "Number of iterations after which p is no longer approximated.", new IntValue(0)));
+      Parameters.Add(new FixedValueParameter<IntValue>(MomentumSwitchIterationParameterName, "Number of iterations after which the momentum in the gradient descent is switched.", new IntValue(0)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(InitialMomentumParameterName, "The initial momentum in the gradient descent.", new DoubleValue(0.5)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(FinalMomentumParameterName, "The final momentum.", new DoubleValue(0.8)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(EtaParameterName, "Gradient descent learning rate.", new DoubleValue(10)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "If the seed should be random.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The seed used if it should not be random.", new IntValue(0)));
+      Parameters.Add(new OptionalConstrainedValueParameter<StringValue>(ClassesNameParameterName, "Name of the column specifying the class lables of each data point. If this is not set training/test is used as labels."));
+      Parameters.Add(new FixedValueParameter<BoolValue>(NormalizationParameterName, "Whether the data should be zero centered and have variance of 1 for each variable, so different scalings are ignored.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<IntValue>(UpdateIntervalParameterName, "The interval after which the results will be updated.", new IntValue(50)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(RandomInitializationParameterName, "Wether data points should be randomly initialized or according to the first 2 dimensions", new BoolValue(true)));
+
+      UpdateIntervalParameter.Hidden = true;
+      MomentumSwitchIterationParameter.Hidden = true;
+      InitialMomentumParameter.Hidden = true;
+      FinalMomentumParameter.Hidden = true;
+      StopLyingIterationParameter.Hidden = true;
+      EtaParameter.Hidden = false;
+      Problem = new RegressionProblem();
+      RegisterParameterEvents();
+    }
+    #endregion
+
+    public override void Prepare() {
+      base.Prepare();
+      dataRowIndices = null;
+      state = null;
+    }
+
+    protected override void Run(CancellationToken cancellationToken) {
+      var problemData = Problem.ProblemData;
+      // set up and initialize everything if necessary
+      var wdist = DistanceFunction as WeightedEuclideanDistance;
+      if (wdist != null) wdist.Initialize(problemData);
+      if (state == null) {
+        if (SetSeedRandomly) Seed = RandomSeedGenerator.GetSeed();
+        var random = new MersenneTwister((uint)Seed);
+        var dataset = problemData.Dataset;
+        var allowedInputVariables = problemData.AllowedInputVariables.ToArray();
+        var allindices = Problem.ProblemData.AllIndices.ToArray();
+
+        // jagged array is required to meet the static method declarations of TSNEStatic<T> 
+        var data = Enumerable.Range(0, dataset.Rows).Select(x => new double[allowedInputVariables.Length]).ToArray();
+        var col = 0;
+        foreach (var s in allowedInputVariables) {
+          var row = 0;
+          foreach (var d in dataset.GetDoubleValues(s)) {
+            data[row][col] = d;
+            row++;
+          }
+          col++;
+        }
+        if (Normalization) data = NormalizeInputData(data);
+        state = TSNEStatic<double[]>.CreateState(data, DistanceFunction, random, NewDimensions, Perplexity, Theta, StopLyingIteration, MomentumSwitchIteration, InitialMomentum, FinalMomentum, Eta, RandomInitialization);
+        SetUpResults(allindices);
+      }
+      while (state.iter < MaxIterations && !cancellationToken.IsCancellationRequested) {
+        if (state.iter % UpdateInterval == 0) Analyze(state);
+        TSNEStatic<double[]>.Iterate(state);
+      }
+      Analyze(state);
+    }
+
+    #region Events
+    protected override void OnProblemChanged() {
+      base.OnProblemChanged();
+      if (Problem == null) return;
+      OnProblemDataChanged(this, null);
+    }
+
+    protected override void RegisterProblemEvents() {
+      base.RegisterProblemEvents();
+      if (Problem == null) return;
+      Problem.ProblemDataChanged += OnProblemDataChanged;
+      if (Problem.ProblemData == null) return;
+      Problem.ProblemData.Changed += OnPerplexityChanged;
+      Problem.ProblemData.Changed += OnColumnsChanged;
+      if (Problem.ProblemData.Dataset == null) return;
+      Problem.ProblemData.Dataset.RowsChanged += OnPerplexityChanged;
+      Problem.ProblemData.Dataset.ColumnsChanged += OnColumnsChanged;
+    }
+
+    protected override void DeregisterProblemEvents() {
+      base.DeregisterProblemEvents();
+      if (Problem == null) return;
+      Problem.ProblemDataChanged -= OnProblemDataChanged;
+      if (Problem.ProblemData == null) return;
+      Problem.ProblemData.Changed -= OnPerplexityChanged;
+      Problem.ProblemData.Changed -= OnColumnsChanged;
+      if (Problem.ProblemData.Dataset == null) return;
+      Problem.ProblemData.Dataset.RowsChanged -= OnPerplexityChanged;
+      Problem.ProblemData.Dataset.ColumnsChanged -= OnColumnsChanged;
+    }
+
+    protected override void OnStopped() {
+      base.OnStopped();
+      //bwerth: state objects can be very large; avoid state serialization 
+      state = null;
+      dataRowIndices = null;
+    }
+
+    private void OnProblemDataChanged(object sender, EventArgs args) {
+      if (Problem == null || Problem.ProblemData == null) return;
+      OnPerplexityChanged(this, null);
+      OnColumnsChanged(this, null);
+      Problem.ProblemData.Changed += OnPerplexityChanged;
+      Problem.ProblemData.Changed += OnColumnsChanged;
+      if (Problem.ProblemData.Dataset == null) return;
+      Problem.ProblemData.Dataset.RowsChanged += OnPerplexityChanged;
+      Problem.ProblemData.Dataset.ColumnsChanged += OnColumnsChanged;
+      if (!Parameters.ContainsKey(ClassesNameParameterName)) return;
+      ClassesNameParameter.ValidValues.Clear();
+      foreach (var input in Problem.ProblemData.InputVariables) ClassesNameParameter.ValidValues.Add(input);
+    }
+
+    private void OnColumnsChanged(object sender, EventArgs e) {
+      if (Problem == null || Problem.ProblemData == null || Problem.ProblemData.Dataset == null || !Parameters.ContainsKey(DistanceFunctionParameterName)) return;
+      DistanceFunctionParameter.ValidValues.OfType<WeightedEuclideanDistance>().Single().AdaptToProblemData(Problem.ProblemData);
+    }
+
+    private void RegisterParameterEvents() {
+      PerplexityParameter.Value.ValueChanged += OnPerplexityChanged;
+    }
+
+    private void OnPerplexityChanged(object sender, EventArgs e) {
+      if (Problem == null || Problem.ProblemData == null || Problem.ProblemData.Dataset == null || !Parameters.ContainsKey(PerplexityParameterName)) return;
+      PerplexityParameter.Value.Value = Math.Max(1, Math.Min((Problem.ProblemData.Dataset.Rows - 1) / 3.0, Perplexity));
+    }
+    #endregion
+
+    #region Helpers
+    private void SetUpResults(IReadOnlyList<int> allIndices) {
+      if (Results == null) return;
+      var results = Results;
+      dataRowIndices = new Dictionary<string, IList<int>>();
+      var problemData = Problem.ProblemData;
+
+      if (!results.ContainsKey(IterationResultName)) results.Add(new Result(IterationResultName, new IntValue(0)));
+      if (!results.ContainsKey(ErrorResultName)) results.Add(new Result(ErrorResultName, new DoubleValue(0)));
+      if (!results.ContainsKey(ScatterPlotResultName)) results.Add(new Result(ScatterPlotResultName, "Plot of the projected data", new ScatterPlot(DataResultName, "")));
+      if (!results.ContainsKey(DataResultName)) results.Add(new Result(DataResultName, "Projected Data", new DoubleMatrix()));
+      if (!results.ContainsKey(ErrorPlotResultName)) {
+        var errortable = new DataTable(ErrorPlotResultName, "Development of errors during gradient descent") {
+          VisualProperties = {
+            XAxisTitle = "UpdateIntervall",
+            YAxisTitle = "Error",
+            YAxisLogScale = true
+          }
+        };
+        errortable.Rows.Add(new DataRow("Errors"));
+        errortable.Rows["Errors"].VisualProperties.StartIndexZero = true;
+        results.Add(new Result(ErrorPlotResultName, errortable));
+      }
+
+      //color datapoints acording to classes variable (be it double, datetime or string)
+      if (!problemData.Dataset.VariableNames.Contains(ClassesName)) {
+        dataRowIndices.Add("Training", problemData.TrainingIndices.ToList());
+        dataRowIndices.Add("Test", problemData.TestIndices.ToList());
+        return;
+      }
+
+      var classificationData = problemData as ClassificationProblemData;
+      if (classificationData != null && classificationData.TargetVariable.Equals(ClassesName)) {
+        var classNames = classificationData.ClassValues.Zip(classificationData.ClassNames, (v, n) => new {v, n}).ToDictionary(x => x.v, x => x.n);
+        var classes = classificationData.Dataset.GetDoubleValues(classificationData.TargetVariable, allIndices).Select(v => classNames[v]).ToArray();
+        for (var i = 0; i < classes.Length; i++) {
+          if (!dataRowIndices.ContainsKey(classes[i])) dataRowIndices.Add(classes[i], new List<int>());
+          dataRowIndices[classes[i]].Add(i);
+        }
+      } else if (((Dataset)problemData.Dataset).VariableHasType<string>(ClassesName)) {
+        var classes = problemData.Dataset.GetStringValues(ClassesName, allIndices).ToArray();
+        for (var i = 0; i < classes.Length; i++) {
+          if (!dataRowIndices.ContainsKey(classes[i])) dataRowIndices.Add(classes[i], new List<int>());
+          dataRowIndices[classes[i]].Add(i);
+        }
+      } else if (((Dataset)problemData.Dataset).VariableHasType<double>(ClassesName)) {
+        var clusterdata = new Dataset(problemData.Dataset.DoubleVariables, problemData.Dataset.DoubleVariables.Select(v => problemData.Dataset.GetDoubleValues(v, allIndices).ToList()));
+        const int contours = 8;
+        Dictionary<int, string> contourMap;
+        IClusteringModel clusterModel;
+        double[][] borders;
+        CreateClusters(clusterdata, ClassesName, contours, out clusterModel, out contourMap, out borders);
+        var contourorder = borders.Select((x, i) => new {x, i}).OrderBy(x => x.x[0]).Select(x => x.i).ToArray();
+        for (var i = 0; i < contours; i++) {
+          var c = contourorder[i];
+          var contourname = contourMap[c];
+          dataRowIndices.Add(contourname, new List<int>());
+          var row = new ScatterPlotDataRow(contourname, "", new List<Point2D<double>>()) {VisualProperties = {Color = GetHeatMapColor(i, contours), PointSize = 8}};
+          ((ScatterPlot)results[ScatterPlotResultName].Value).Rows.Add(row);
+        }
+        var allClusters = clusterModel.GetClusterValues(clusterdata, Enumerable.Range(0, clusterdata.Rows)).ToArray();
+        for (var i = 0; i < clusterdata.Rows; i++) dataRowIndices[contourMap[allClusters[i] - 1]].Add(i);
+      } else if (((Dataset)problemData.Dataset).VariableHasType<DateTime>(ClassesName)) {
+        var clusterdata = new Dataset(problemData.Dataset.DateTimeVariables, problemData.Dataset.DateTimeVariables.Select(v => problemData.Dataset.GetDoubleValues(v, allIndices).ToList()));
+        const int contours = 8;
+        Dictionary<int, string> contourMap;
+        IClusteringModel clusterModel;
+        double[][] borders;
+        CreateClusters(clusterdata, ClassesName, contours, out clusterModel, out contourMap, out borders);
+        var contourorder = borders.Select((x, i) => new {x, i}).OrderBy(x => x.x[0]).Select(x => x.i).ToArray();
+        for (var i = 0; i < contours; i++) {
+          var c = contourorder[i];
+          var contourname = contourMap[c];
+          dataRowIndices.Add(contourname, new List<int>());
+          var row = new ScatterPlotDataRow(contourname, "", new List<Point2D<double>>()) {VisualProperties = {Color = GetHeatMapColor(i, contours), PointSize = 8}};
+          row.VisualProperties.PointSize = 8;
+          ((ScatterPlot)results[ScatterPlotResultName].Value).Rows.Add(row);
+        }
+        var allClusters = clusterModel.GetClusterValues(clusterdata, Enumerable.Range(0, clusterdata.Rows)).ToArray();
+        for (var i = 0; i < clusterdata.Rows; i++) dataRowIndices[contourMap[allClusters[i] - 1]].Add(i);
+      } else {
+        dataRowIndices.Add("Training", problemData.TrainingIndices.ToList());
+        dataRowIndices.Add("Test", problemData.TestIndices.ToList());
+      }
+    }
+
+    private void Analyze(TSNEStatic<double[]>.TSNEState tsneState) {
+      if (Results == null) return;
+      var results = Results;
+      var plot = results[ErrorPlotResultName].Value as DataTable;
+      if (plot == null) throw new ArgumentException("Could not create/access error data table in results collection.");
+      var errors = plot.Rows["Errors"].Values;
+      var c = tsneState.EvaluateError();
+      errors.Add(c);
+      ((IntValue)results[IterationResultName].Value).Value = tsneState.iter;
+      ((DoubleValue)results[ErrorResultName].Value).Value = errors.Last();
+
+      var ndata = NormalizeProjectedData(tsneState.newData);
+      results[DataResultName].Value = new DoubleMatrix(ndata);
+      var splot = results[ScatterPlotResultName].Value as ScatterPlot;
+      FillScatterPlot(ndata, splot, dataRowIndices);
+    }
+
+    private static void FillScatterPlot(double[,] lowDimData, ScatterPlot plot, Dictionary<string, IList<int>> dataRowIndices) {
+      foreach (var rowName in dataRowIndices.Keys) {
+        if (!plot.Rows.ContainsKey(rowName)) {
+          plot.Rows.Add(new ScatterPlotDataRow(rowName, "", new List<Point2D<double>>()));
+          plot.Rows[rowName].VisualProperties.PointSize = 8;
+        }
+        plot.Rows[rowName].Points.Replace(dataRowIndices[rowName].Select(i => new Point2D<double>(lowDimData[i, 0], lowDimData[i, 1])));
+      }
+    }
+
+    private static double[,] NormalizeProjectedData(double[,] data) {
+      var max = new double[data.GetLength(1)];
+      var min = new double[data.GetLength(1)];
+      var res = new double[data.GetLength(0), data.GetLength(1)];
+      for (var i = 0; i < max.Length; i++) max[i] = min[i] = data[0, i];
+      for (var i = 0; i < data.GetLength(0); i++)
+      for (var j = 0; j < data.GetLength(1); j++) {
+        var v = data[i, j];
+        max[j] = Math.Max(max[j], v);
+        min[j] = Math.Min(min[j], v);
+      }
+      for (var i = 0; i < data.GetLength(0); i++) {
+        for (var j = 0; j < data.GetLength(1); j++) {
+          var d = max[j] - min[j];
+          var s = data[i, j] - (max[j] + min[j]) / 2; //shift data
+          if (d.IsAlmost(0)) res[i, j] = data[i, j]; //no scaling possible
+          else res[i, j] = s / d; //scale data
+        }
+      }
+      return res;
+    }
+
+    private static double[][] NormalizeInputData(IReadOnlyList<IReadOnlyList<double>> data) {
+      // as in tSNE implementation by van der Maaten
+      var n = data[0].Count;
+      var mean = new double[n];
+      var max = new double[n];
+      var nData = new double[data.Count][];
+      for (var i = 0; i < n; i++) {
+        mean[i] = Enumerable.Range(0, data.Count).Select(x => data[x][i]).Average();
+        max[i] = Enumerable.Range(0, data.Count).Max(x => Math.Abs(data[x][i]));
+      }
+      for (var i = 0; i < data.Count; i++) {
+        nData[i] = new double[n];
+        for (var j = 0; j < n; j++)
+          nData[i][j] = max[j].IsAlmost(0) ? data[i][j] - mean[j] : (data[i][j] - mean[j]) / max[j];
+      }
+      return nData;
+    }
+
+    private static Color GetHeatMapColor(int contourNr, int noContours) {
+      return ConvertTotalToRgb(0, noContours, contourNr);
+    }
+
+    private static void CreateClusters(IDataset data, string target, int contours, out IClusteringModel contourCluster, out Dictionary<int, string> contourNames, out double[][] borders) {
+      var cpd = new ClusteringProblemData((Dataset)data, new[] {target});
+      contourCluster = KMeansClustering.CreateKMeansSolution(cpd, contours, 3).Model;
+
+      borders = Enumerable.Range(0, contours).Select(x => new[] {double.MaxValue, double.MinValue}).ToArray();
+      var clusters = contourCluster.GetClusterValues(cpd.Dataset, cpd.AllIndices).ToArray();
+      var targetvalues = cpd.Dataset.GetDoubleValues(target).ToArray();
+      foreach (var i in cpd.AllIndices) {
+        var cl = clusters[i] - 1;
+        var clv = targetvalues[i];
+        if (borders[cl][0] > clv) borders[cl][0] = clv;
+        if (borders[cl][1] < clv) borders[cl][1] = clv;
+      }
+
+      contourNames = new Dictionary<int, string>();
+      for (var i = 0; i < contours; i++)
+        contourNames.Add(i, "[" + borders[i][0] + ";" + borders[i][1] + "]");
+    }
+
+    private static Color ConvertTotalToRgb(double low, double high, double cell) {
+      var colorGradient = ColorGradient.Colors;
+      var range = high - low;
+      var h = Math.Min(cell / range * colorGradient.Count, colorGradient.Count - 1);
+      return colorGradient[(int)h];
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/TSNEStatic.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/TSNEStatic.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/TSNEStatic.cs	(revision 17041)
@@ -0,0 +1,731 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//Code is based on an implementation from Laurens van der Maaten
+
+/*
+*
+* Copyright (c) 2014, Laurens van der Maaten (Delft University of Technology)
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following conditions are met:
+* 1. Redistributions of source code must retain the above copyright
+*    notice, this list of conditions and the following disclaimer.
+* 2. Redistributions in binary form must reproduce the above copyright
+*    notice, this list of conditions and the following disclaimer in the
+*    documentation and/or other materials provided with the distribution.
+* 3. All advertising materials mentioning features or use of this software
+*    must display the following acknowledgement:
+*    This product includes software developed by the Delft University of Technology.
+* 4. Neither the name of the Delft University of Technology nor the names of
+*    its contributors may be used to endorse or promote products derived from
+*    this software without specific prior written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY LAURENS VAN DER MAATEN ''AS IS'' AND ANY EXPRESS
+* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+* EVENT SHALL LAURENS VAN DER MAATEN BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+* OF SUCH DAMAGE.
+*
+*/
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Collections;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Random;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("23E56F69-7AA1-4853-BFEC-7B4DBC346B47")]
+  public class TSNEStatic<T> {
+    [StorableConstructor]
+    protected TSNEStatic(StorableConstructorFlag _) {
+    }
+
+    [StorableType("1878DFC2-5DDF-4BBD-85BA-4A6E19AB59C2")]
+    public sealed class TSNEState : DeepCloneable {
+      #region Storables
+      // initialized once
+      [Storable]
+      public IDistance<T> distance;
+      [Storable]
+      public IRandom random;
+      [Storable]
+      public double perplexity;
+      [Storable]
+      public bool exact;
+      [Storable]
+      public int noDatapoints;
+      [Storable]
+      public double finalMomentum;
+      [Storable]
+      public int momSwitchIter;
+      [Storable]
+      public int stopLyingIter;
+      [Storable]
+      public double theta;
+      [Storable]
+      public double eta;
+      [Storable]
+      public int newDimensions;
+
+      // for approximate version: sparse representation of similarity/distance matrix
+      [Storable]
+      public double[] valP; // similarity/distance
+      [Storable]
+      public int[] rowP; // row index
+      [Storable]
+      public int[] colP; // col index
+
+      // for exact version: dense representation of distance/similarity matrix
+      [Storable]
+      public double[,] p;
+
+      // mapped data
+      [Storable]
+      public double[,] newData;
+
+      [Storable]
+      public int iter;
+      [Storable]
+      public double currentMomentum;
+
+      // helper variables (updated in each iteration)
+      [Storable]
+      public double[,] gains;
+      [Storable]
+      public double[,] uY;
+      [Storable]
+      public double[,] dY;
+      #endregion
+
+      #region Constructors & Cloning
+      private TSNEState(TSNEState original, Cloner cloner) : base(original, cloner) {
+        distance = cloner.Clone(original.distance);
+        random = cloner.Clone(original.random);
+        perplexity = original.perplexity;
+        exact = original.exact;
+        noDatapoints = original.noDatapoints;
+        finalMomentum = original.finalMomentum;
+        momSwitchIter = original.momSwitchIter;
+        stopLyingIter = original.stopLyingIter;
+        theta = original.theta;
+        eta = original.eta;
+        newDimensions = original.newDimensions;
+        if (original.valP != null) {
+          valP = new double[original.valP.Length];
+          Array.Copy(original.valP, valP, valP.Length);
+        }
+        if (original.rowP != null) {
+          rowP = new int[original.rowP.Length];
+          Array.Copy(original.rowP, rowP, rowP.Length);
+        }
+        if (original.colP != null) {
+          colP = new int[original.colP.Length];
+          Array.Copy(original.colP, colP, colP.Length);
+        }
+        if (original.p != null) {
+          p = new double[original.p.GetLength(0), original.p.GetLength(1)];
+          Array.Copy(original.p, p, p.Length);
+        }
+        newData = new double[original.newData.GetLength(0), original.newData.GetLength(1)];
+        Array.Copy(original.newData, newData, newData.Length);
+        iter = original.iter;
+        currentMomentum = original.currentMomentum;
+        gains = new double[original.gains.GetLength(0), original.gains.GetLength(1)];
+        Array.Copy(original.gains, gains, gains.Length);
+        uY = new double[original.uY.GetLength(0), original.uY.GetLength(1)];
+        Array.Copy(original.uY, uY, uY.Length);
+        dY = new double[original.dY.GetLength(0), original.dY.GetLength(1)];
+        Array.Copy(original.dY, dY, dY.Length);
+      }
+
+      public override IDeepCloneable Clone(Cloner cloner) {
+        return new TSNEState(this, cloner);
+      }
+
+      [StorableConstructor]
+      private TSNEState(StorableConstructorFlag _) { }
+
+      public TSNEState(IReadOnlyList<T> data, IDistance<T> distance, IRandom random, int newDimensions, double perplexity,
+        double theta, int stopLyingIter, int momSwitchIter, double momentum, double finalMomentum, double eta, bool randomInit) {
+        this.distance = distance;
+        this.random = random;
+        this.newDimensions = newDimensions;
+        this.perplexity = perplexity;
+        this.theta = theta;
+        this.stopLyingIter = stopLyingIter;
+        this.momSwitchIter = momSwitchIter;
+        currentMomentum = momentum;
+        this.finalMomentum = finalMomentum;
+        this.eta = eta;
+
+        // initialize
+        noDatapoints = data.Count;
+        if (noDatapoints - 1 < 3 * perplexity)
+          throw new ArgumentException("Perplexity too large for the number of data points!");
+
+        exact = Math.Abs(theta) < double.Epsilon;
+        newData = new double[noDatapoints, newDimensions];
+        dY = new double[noDatapoints, newDimensions];
+        uY = new double[noDatapoints, newDimensions];
+        gains = new double[noDatapoints, newDimensions];
+        for (var i = 0; i < noDatapoints; i++)
+        for (var j = 0; j < newDimensions; j++)
+          gains[i, j] = 1.0;
+
+        p = null;
+        rowP = null;
+        colP = null;
+        valP = null;
+
+        //Calculate Similarities
+        if (exact) p = CalculateExactSimilarites(data, distance, perplexity);
+        else CalculateApproximateSimilarities(data, distance, perplexity, out rowP, out colP, out valP);
+
+        // Lie about the P-values (factor is 4 in the MATLAB implementation)
+        if (exact) for (var i = 0; i < noDatapoints; i++) for (var j = 0; j < noDatapoints; j++) p[i, j] *= 12.0;
+        else for (var i = 0; i < rowP[noDatapoints]; i++) valP[i] *= 12.0;
+
+        // Initialize solution (randomly)
+        var rand = new NormalDistributedRandom(random, 0, 1);
+        for (var i = 0; i < noDatapoints; i++)
+        for (var j = 0; j < newDimensions; j++)
+          newData[i, j] = rand.NextDouble() * .0001;
+
+        if (!(data[0] is IReadOnlyList<double>) || randomInit) return;
+        for (var i = 0; i < noDatapoints; i++)
+        for (var j = 0; j < newDimensions; j++) {
+          var row = (IReadOnlyList<double>) data[i];
+          newData[i, j] = row[j % row.Count];
+        }
+      }
+      #endregion
+
+      public double EvaluateError() {
+        return exact ? EvaluateErrorExact(p, newData, noDatapoints, newDimensions) : EvaluateErrorApproximate(rowP, colP, valP, newData, theta);
+      }
+
+      #region Helpers
+      private static void CalculateApproximateSimilarities(IReadOnlyList<T> data, IDistance<T> distance, double perplexity, out int[] rowP, out int[] colP, out double[] valP) {
+        // Compute asymmetric pairwise input similarities
+        ComputeGaussianPerplexity(data, distance, out rowP, out colP, out valP, perplexity, (int) (3 * perplexity));
+        // Symmetrize input similarities
+        int[] sRowP, symColP;
+        double[] sValP;
+        SymmetrizeMatrix(rowP, colP, valP, out sRowP, out symColP, out sValP);
+        rowP = sRowP;
+        colP = symColP;
+        valP = sValP;
+        var sumP = .0;
+        for (var i = 0; i < rowP[data.Count]; i++) sumP += valP[i];
+        for (var i = 0; i < rowP[data.Count]; i++) valP[i] /= sumP;
+      }
+      private static double[,] CalculateExactSimilarites(IReadOnlyList<T> data, IDistance<T> distance, double perplexity) {
+        // Compute similarities
+        var p = new double[data.Count, data.Count];
+        ComputeGaussianPerplexity(data, distance, p, perplexity);
+        // Symmetrize input similarities
+        for (var n = 0; n < data.Count; n++) {
+          for (var m = n + 1; m < data.Count; m++) {
+            p[n, m] += p[m, n];
+            p[m, n] = p[n, m];
+          }
+        }
+        var sumP = .0;
+        for (var i = 0; i < data.Count; i++) {
+          for (var j = 0; j < data.Count; j++) {
+            sumP += p[i, j];
+          }
+        }
+        for (var i = 0; i < data.Count; i++) {
+          for (var j = 0; j < data.Count; j++) {
+            p[i, j] /= sumP;
+          }
+        }
+        return p;
+      }
+      private static void ComputeGaussianPerplexity(IReadOnlyList<T> x, IDistance<T> distance, out int[] rowP, out int[] colP, out double[] valP, double perplexity, int k) {
+        if (perplexity > k) throw new ArgumentException("Perplexity should be lower than k!");
+
+        var n = x.Count;
+        // Allocate the memory we need
+        rowP = new int[n + 1];
+        colP = new int[n * k];
+        valP = new double[n * k];
+        var curP = new double[n - 1];
+        rowP[0] = 0;
+        for (var i = 0; i < n; i++) rowP[i + 1] = rowP[i] + k;
+
+        var objX = new List<IndexedItem<T>>();
+        for (var i = 0; i < n; i++) objX.Add(new IndexedItem<T>(i, x[i]));
+
+        // Build ball tree on data set
+        var tree = new VantagePointTree<IndexedItem<T>>(new IndexedItemDistance<T>(distance), objX);
+
+        // Loop over all points to find nearest neighbors
+        for (var i = 0; i < n; i++) {
+          IList<IndexedItem<T>> indices;
+          IList<double> distances;
+
+          // Find nearest neighbors
+          tree.Search(objX[i], k + 1, out indices, out distances);
+
+          // Initialize some variables for binary search
+          var found = false;
+          var beta = 1.0;
+          var minBeta = double.MinValue;
+          var maxBeta = double.MaxValue;
+          const double tol = 1e-5;
+
+          // Iterate until we found a good perplexity
+          var iter = 0;
+          double sumP = 0;
+          while (!found && iter < 200) {
+            // Compute Gaussian kernel row
+            for (var m = 0; m < k; m++) curP[m] = Math.Exp(-beta * distances[m + 1]);
+
+            // Compute entropy of current row
+            sumP = double.Epsilon;
+            for (var m = 0; m < k; m++) sumP += curP[m];
+            var h = .0;
+            for (var m = 0; m < k; m++) h += beta * (distances[m + 1] * curP[m]);
+            h = h / sumP + Math.Log(sumP);
+
+            // Evaluate whether the entropy is within the tolerance level
+            var hdiff = h - Math.Log(perplexity);
+            if (hdiff < tol && -hdiff < tol) {
+              found = true;
+            }
+            else {
+              if (hdiff > 0) {
+                minBeta = beta;
+                if (maxBeta.IsAlmost(double.MaxValue) || maxBeta.IsAlmost(double.MinValue))
+                  beta *= 2.0;
+                else
+                  beta = (beta + maxBeta) / 2.0;
+              }
+              else {
+                maxBeta = beta;
+                if (minBeta.IsAlmost(double.MinValue) || minBeta.IsAlmost(double.MaxValue))
+                  beta /= 2.0;
+                else
+                  beta = (beta + minBeta) / 2.0;
+              }
+            }
+
+            // Update iteration counter
+            iter++;
+          }
+
+          // Row-normalize current row of P and store in matrix
+          for (var m = 0; m < k; m++) curP[m] /= sumP;
+          for (var m = 0; m < k; m++) {
+            colP[rowP[i] + m] = indices[m + 1].Index;
+            valP[rowP[i] + m] = curP[m];
+          }
+        }
+      }
+      private static void ComputeGaussianPerplexity(IReadOnlyList<T> x, IDistance<T> distance, double[,] p, double perplexity) {
+        // Compute the distance matrix
+        var dd = ComputeDistances(x, distance);
+
+        var n = x.Count;
+        // Compute the Gaussian kernel row by row
+        for (var i = 0; i < n; i++) {
+          // Initialize some variables
+          var found = false;
+          var beta = 1.0;
+          var minBeta = double.MinValue;
+          var maxBeta = double.MaxValue;
+          const double tol = 1e-5;
+          double sumP = 0;
+
+          // Iterate until we found a good perplexity
+          var iter = 0;
+          while (!found && iter < 200) { // 200 iterations as in tSNE implementation by van der Maarten
+
+            // Compute Gaussian kernel row
+            for (var m = 0; m < n; m++) p[i, m] = Math.Exp(-beta * dd[i][m]);
+            p[i, i] = double.Epsilon;
+
+            // Compute entropy of current row
+            sumP = double.Epsilon;
+            for (var m = 0; m < n; m++) sumP += p[i, m];
+            var h = 0.0;
+            for (var m = 0; m < n; m++) h += beta * (dd[i][m] * p[i, m]);
+            h = h / sumP + Math.Log(sumP);
+
+            // Evaluate whether the entropy is within the tolerance level
+            var hdiff = h - Math.Log(perplexity);
+            if (hdiff < tol && -hdiff < tol) {
+              found = true;
+            }
+            else {
+              if (hdiff > 0) {
+                minBeta = beta;
+                if (maxBeta.IsAlmost(double.MaxValue) || maxBeta.IsAlmost(double.MinValue))
+                  beta *= 2.0;
+                else
+                  beta = (beta + maxBeta) / 2.0;
+              }
+              else {
+                maxBeta = beta;
+                if (minBeta.IsAlmost(double.MinValue) || minBeta.IsAlmost(double.MaxValue))
+                  beta /= 2.0;
+                else
+                  beta = (beta + minBeta) / 2.0;
+              }
+            }
+
+            // Update iteration counter
+            iter++;
+          }
+
+          // Row normalize P
+          for (var m = 0; m < n; m++) p[i, m] /= sumP;
+        }
+      }
+      private static double[][] ComputeDistances(IReadOnlyList<T> x, IDistance<T> distance) {
+        var res = new double[x.Count][];
+        for (var r = 0; r < x.Count; r++) {
+          var rowV = new double[x.Count];
+          // all distances must be symmetric 
+          for (var c = 0; c < r; c++) {
+            rowV[c] = res[c][r];
+          }
+          rowV[r] = 0.0; // distance to self is zero for all distances
+          for (var c = r + 1; c < x.Count; c++) {
+            rowV[c] = distance.Get(x[r], x[c]);
+          }
+          res[r] = rowV;
+        }
+        return res;
+        // return x.Select(m => x.Select(n => distance.Get(m, n)).ToArray()).ToArray();
+      }
+      private static double EvaluateErrorExact(double[,] p, double[,] y, int n, int d) {
+        // Compute the squared Euclidean distance matrix
+        var dd = new double[n, n];
+        var q = new double[n, n];
+        ComputeSquaredEuclideanDistance(y, n, d, dd);
+
+        // Compute Q-matrix and normalization sum
+        var sumQ = double.Epsilon;
+        for (var n1 = 0; n1 < n; n1++) {
+          for (var m = 0; m < n; m++) {
+            if (n1 != m) {
+              q[n1, m] = 1 / (1 + dd[n1, m]);
+              sumQ += q[n1, m];
+            }
+            else q[n1, m] = double.Epsilon;
+          }
+        }
+        for (var i = 0; i < n; i++) for (var j = 0; j < n; j++) q[i, j] /= sumQ;
+
+        // Sum t-SNE error
+        var c = .0;
+        for (var i = 0; i < n; i++)
+        for (var j = 0; j < n; j++) {
+          c += p[i, j] * Math.Log((p[i, j] + float.Epsilon) / (q[i, j] + float.Epsilon));
+        }
+        return c;
+      }
+      private static double EvaluateErrorApproximate(IReadOnlyList<int> rowP, IReadOnlyList<int> colP, IReadOnlyList<double> valP, double[,] y, double theta) {
+        // Get estimate of normalization term
+        var n = y.GetLength(0);
+        var d = y.GetLength(1);
+        var tree = new SpacePartitioningTree(y);
+        var buff = new double[d];
+        var sumQ = 0.0;
+        for (var i = 0; i < n; i++) tree.ComputeNonEdgeForces(i, theta, buff, ref sumQ);
+
+        // Loop over all edges to compute t-SNE error
+        var c = .0;
+        for (var k = 0; k < n; k++) {
+          for (var i = rowP[k]; i < rowP[k + 1]; i++) {
+            var q = .0;
+            for (var j = 0; j < d; j++) buff[j] = y[k, j];
+            for (var j = 0; j < d; j++) buff[j] -= y[colP[i], j];
+            for (var j = 0; j < d; j++) q += buff[j] * buff[j];
+            q = (1.0 / (1.0 + q)) / sumQ;
+            c += valP[i] * Math.Log((valP[i] + float.Epsilon) / (q + float.Epsilon));
+          }
+        }
+        return c;
+      }
+      private static void SymmetrizeMatrix(IReadOnlyList<int> rowP, IReadOnlyList<int> colP, IReadOnlyList<double> valP, out int[] symRowP, out int[] symColP, out double[] symValP) {
+        // Count number of elements and row counts of symmetric matrix
+        var n = rowP.Count - 1;
+        var rowCounts = new int[n];
+        for (var j = 0; j < n; j++) {
+          for (var i = rowP[j]; i < rowP[j + 1]; i++) {
+            // Check whether element (col_P[i], n) is present
+            var present = false;
+            for (var m = rowP[colP[i]]; m < rowP[colP[i] + 1]; m++) {
+              if (colP[m] == j) present = true;
+            }
+            if (present) rowCounts[j]++;
+            else {
+              rowCounts[j]++;
+              rowCounts[colP[i]]++;
+            }
+          }
+        }
+        var noElem = 0;
+        for (var i = 0; i < n; i++) noElem += rowCounts[i];
+
+        // Allocate memory for symmetrized matrix
+        symRowP = new int[n + 1];
+        symColP = new int[noElem];
+        symValP = new double[noElem];
+
+        // Construct new row indices for symmetric matrix
+        symRowP[0] = 0;
+        for (var i = 0; i < n; i++) symRowP[i + 1] = symRowP[i] + rowCounts[i];
+
+        // Fill the result matrix
+        var offset = new int[n];
+        for (var j = 0; j < n; j++) {
+          for (var i = rowP[j]; i < rowP[j + 1]; i++) { // considering element(n, colP[i])
+
+            // Check whether element (col_P[i], n) is present
+            var present = false;
+            for (var m = rowP[colP[i]]; m < rowP[colP[i] + 1]; m++) {
+              if (colP[m] != j) continue;
+              present = true;
+              if (j > colP[i]) continue; // make sure we do not add elements twice
+              symColP[symRowP[j] + offset[j]] = colP[i];
+              symColP[symRowP[colP[i]] + offset[colP[i]]] = j;
+              symValP[symRowP[j] + offset[j]] = valP[i] + valP[m];
+              symValP[symRowP[colP[i]] + offset[colP[i]]] = valP[i] + valP[m];
+            }
+
+            // If (colP[i], n) is not present, there is no addition involved
+            if (!present) {
+              symColP[symRowP[j] + offset[j]] = colP[i];
+              symColP[symRowP[colP[i]] + offset[colP[i]]] = j;
+              symValP[symRowP[j] + offset[j]] = valP[i];
+              symValP[symRowP[colP[i]] + offset[colP[i]]] = valP[i];
+            }
+
+            // Update offsets
+            if (present && (j > colP[i])) continue;
+            offset[j]++;
+            if (colP[i] != j) offset[colP[i]]++;
+          }
+        }
+
+        for (var i = 0; i < noElem; i++) symValP[i] /= 2.0;
+      }
+      #endregion
+    }
+
+    /// <summary>
+    /// Static interface to tSNE
+    /// </summary>
+    /// <param name="data"></param>
+    /// <param name="distance">The distance function used to differentiate similar from non-similar points, e.g. Euclidean distance.</param>
+    /// <param name="random">Random number generator</param>
+    /// <param name="newDimensions">Dimensionality of projected space (usually 2 for easy visual analysis).</param>
+    /// <param name="perplexity">Perplexity parameter of tSNE. Comparable to k in a k-nearest neighbour algorithm. Recommended value is floor(number of points /3) or lower</param>
+    /// <param name="iterations">Maximum number of iterations for gradient descent.</param>
+    /// <param name="theta">Value describing how much appoximated gradients my differ from exact gradients. Set to 0 for exact calculation and in [0,1] otherwise. CAUTION: exact calculation of forces requires building a non-sparse N*N matrix where N is the number of data points. This may exceed memory limitations.</param>
+    /// <param name="stopLyingIter">Number of iterations after which p is no longer approximated.</param>
+    /// <param name="momSwitchIter">Number of iterations after which the momentum in the gradient descent is switched.</param>
+    /// <param name="momentum">The initial momentum in the gradient descent.</param>
+    /// <param name="finalMomentum">The final momentum in gradient descent (after momentum switch).</param>
+    /// <param name="eta">Gradient descent learning rate.</param>
+    /// <returns></returns>
+    public static double[,] Run(T[] data, IDistance<T> distance, IRandom random,
+      int newDimensions = 2, double perplexity = 25, int iterations = 1000,
+      double theta = 0, int stopLyingIter = 0, int momSwitchIter = 0, double momentum = .5,
+      double finalMomentum = .8, double eta = 10.0
+    ) {
+      var state = CreateState(data, distance, random, newDimensions, perplexity,
+        theta, stopLyingIter, momSwitchIter, momentum, finalMomentum, eta);
+
+      for (var i = 0; i < iterations - 1; i++) {
+        Iterate(state);
+      }
+      return Iterate(state);
+    }
+
+    public static TSNEState CreateState(T[] data, IDistance<T> distance, IRandom random,
+      int newDimensions = 2, double perplexity = 25, double theta = 0,
+      int stopLyingIter = 0, int momSwitchIter = 0, double momentum = .5,
+      double finalMomentum = .8, double eta = 10.0, bool randomInit = true
+    ) {
+      return new TSNEState(data, distance, random, newDimensions, perplexity, theta, stopLyingIter, momSwitchIter, momentum, finalMomentum, eta, randomInit);
+    }
+
+    public static double[,] Iterate(TSNEState state) {
+      if (state.exact)
+        ComputeExactGradient(state.p, state.newData, state.noDatapoints, state.newDimensions, state.dY);
+      else
+        ComputeApproximateGradient(state.rowP, state.colP, state.valP, state.newData, state.noDatapoints, state.newDimensions, state.dY, state.theta);
+
+      // Update gains
+      for (var i = 0; i < state.noDatapoints; i++) {
+        for (var j = 0; j < state.newDimensions; j++) {
+          state.gains[i, j] = Math.Sign(state.dY[i, j]) != Math.Sign(state.uY[i, j])
+            ? state.gains[i, j] + .2 // +0.2 nd *0.8 are used in two separate implementations of tSNE -> seems to be correct
+            : state.gains[i, j] * .8;
+          if (state.gains[i, j] < .01) state.gains[i, j] = .01;
+        }
+      }
+
+      // Perform gradient update (with momentum and gains)
+      for (var i = 0; i < state.noDatapoints; i++)
+      for (var j = 0; j < state.newDimensions; j++)
+        state.uY[i, j] = state.currentMomentum * state.uY[i, j] - state.eta * state.gains[i, j] * state.dY[i, j];
+
+      for (var i = 0; i < state.noDatapoints; i++)
+      for (var j = 0; j < state.newDimensions; j++)
+        state.newData[i, j] = state.newData[i, j] + state.uY[i, j];
+
+      // Make solution zero-mean
+      ZeroMean(state.newData);
+
+      // Stop lying about the P-values after a while, and switch momentum
+      if (state.iter == state.stopLyingIter) {
+        if (state.exact)
+          for (var i = 0; i < state.noDatapoints; i++)
+          for (var j = 0; j < state.noDatapoints; j++)
+            state.p[i, j] /= 12.0;
+        else
+          for (var i = 0; i < state.rowP[state.noDatapoints]; i++)
+            state.valP[i] /= 12.0;
+      }
+
+      if (state.iter == state.momSwitchIter)
+        state.currentMomentum = state.finalMomentum;
+
+      state.iter++;
+      return state.newData;
+    }
+
+    #region Helpers
+    private static void ComputeApproximateGradient(int[] rowP, int[] colP, double[] valP, double[,] y, int n, int d, double[,] dC, double theta) {
+      var tree = new SpacePartitioningTree(y);
+      var sumQ = 0.0;
+      var posF = new double[n, d];
+      var negF = new double[n, d];
+      SpacePartitioningTree.ComputeEdgeForces(rowP, colP, valP, n, posF, y, d);
+      var row = new double[d];
+      for (var n1 = 0; n1 < n; n1++) {
+        Array.Clear(row, 0, row.Length);
+        tree.ComputeNonEdgeForces(n1, theta, row, ref sumQ);
+        Buffer.BlockCopy(row, 0, negF, (sizeof(double) * n1 * d), d * sizeof(double));
+      }
+
+      // Compute final t-SNE gradient
+      for (var i = 0; i < n; i++)
+      for (var j = 0; j < d; j++) {
+        dC[i, j] = posF[i, j] - negF[i, j] / sumQ;
+      }
+    }
+
+    private static void ComputeExactGradient(double[,] p, double[,] y, int n, int d, double[,] dC) {
+      // Make sure the current gradient contains zeros
+      for (var i = 0; i < n; i++) for (var j = 0; j < d; j++) dC[i, j] = 0.0;
+
+      // Compute the squared Euclidean distance matrix
+      var dd = new double[n, n];
+      ComputeSquaredEuclideanDistance(y, n, d, dd);
+
+      // Compute Q-matrix and normalization sum
+      var q = new double[n, n];
+      var sumQ = .0;
+      for (var n1 = 0; n1 < n; n1++) {
+        for (var m = 0; m < n; m++) {
+          if (n1 == m) continue;
+          q[n1, m] = 1 / (1 + dd[n1, m]);
+          sumQ += q[n1, m];
+        }
+      }
+
+      // Perform the computation of the gradient
+      for (var n1 = 0; n1 < n; n1++) {
+        for (var m = 0; m < n; m++) {
+          if (n1 == m) continue;
+          var mult = (p[n1, m] - q[n1, m] / sumQ) * q[n1, m];
+          for (var d1 = 0; d1 < d; d1++) {
+            dC[n1, d1] += (y[n1, d1] - y[m, d1]) * mult;
+          }
+        }
+      }
+    }
+
+    private static void ComputeSquaredEuclideanDistance(double[,] x, int n, int d, double[,] dd) {
+      var dataSums = new double[n];
+      for (var i = 0; i < n; i++) {
+        for (var j = 0; j < d; j++) {
+          dataSums[i] += x[i, j] * x[i, j];
+        }
+      }
+      for (var i = 0; i < n; i++) {
+        for (var m = 0; m < n; m++) {
+          dd[i, m] = dataSums[i] + dataSums[m];
+        }
+      }
+      for (var i = 0; i < n; i++) {
+        dd[i, i] = 0.0;
+        for (var m = i + 1; m < n; m++) {
+          dd[i, m] = 0.0;
+          for (var j = 0; j < d; j++) {
+            dd[i, m] += (x[i, j] - x[m, j]) * (x[i, j] - x[m, j]);
+          }
+          dd[m, i] = dd[i, m];
+        }
+      }
+    }
+
+    private static void ZeroMean(double[,] x) {
+      // Compute data mean
+      var n = x.GetLength(0);
+      var d = x.GetLength(1);
+      var mean = new double[d];
+      for (var i = 0; i < n; i++) {
+        for (var j = 0; j < d; j++) {
+          mean[j] += x[i, j];
+        }
+      }
+      for (var i = 0; i < d; i++) {
+        mean[i] /= n;
+      }
+      // Subtract data mean
+      for (var i = 0; i < n; i++) {
+        for (var j = 0; j < d; j++) {
+          x[i, j] -= mean[j];
+        }
+      }
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/TSNEUtils.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/TSNEUtils.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/TSNEUtils.cs	(revision 17041)
@@ -0,0 +1,79 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  internal static class TSNEUtils {
+    internal static void ForEach<T>(this IEnumerable<T> sequence, Action<int, T> action) {
+      if (sequence == null) return;
+      if (action == null) throw new ArgumentException("Null Action can not be performed");
+      var i = 0;
+      foreach (var item in sequence) {
+        action(i, item);
+        i++;
+      }
+    }
+
+    internal static void Swap<T>(this IList<T> list, int indexA, int indexB) {
+      var tmp = list[indexA];
+      list[indexA] = list[indexB];
+      list[indexB] = tmp;
+    }
+
+    private static int Partition<T>(this IList<T> list, int left, int right, int pivotindex, IComparer<T> comparer) {
+      var pivotValue = list[pivotindex];
+      list.Swap(pivotindex, right);
+      var storeIndex = left;
+      for (var i = left; i < right; i++)
+        if (comparer.Compare(list[i], pivotValue) < 0)
+          list.Swap(storeIndex++, i);
+      list.Swap(right, storeIndex);
+      return storeIndex;
+    }
+
+    /// <summary>
+    /// Quick-Sort based partial ascending sorting
+    /// [0,left-1] not changed;
+    /// [left, n] elements are smaller than or equal to list[n]
+    /// [n, right] elements are grater than or equal to list[n]
+    /// [right, list.Count[ not changed;
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    /// <param name="list">list that shall be partially sorted</param>
+    /// <param name="left">left index of sorted list part</param>
+    /// <param name="right">right index of sorted list part</param>
+    /// <param name="n">index around which the partial sorting occures</param>
+    /// <param name="comparer">comparer for list elemnts </param>
+    /// <returns></returns>
+    internal static void PartialSort<T>(this IList<T> list, int left, int right, int n, IComparer<T> comparer) {
+      while (true) {
+        if (left == right) return;
+        var pivotindex = left + (int) Math.Floor(new System.Random().Next() % (right - (double) left + 1));
+        pivotindex = list.Partition(left, right, pivotindex, comparer);
+        if (n == pivotindex) return;
+        if (n < pivotindex) right = pivotindex - 1;
+        else left = pivotindex + 1;
+      }
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/VantagePointTree.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/VantagePointTree.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/VantagePointTree.cs	(revision 17041)
@@ -0,0 +1,169 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//Code is based on an implementation from Laurens van der Maaten
+
+/*
+*
+* Copyright (c) 2014, Laurens van der Maaten (Delft University of Technology)
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following conditions are met:
+* 1. Redistributions of source code must retain the above copyright
+*    notice, this list of conditions and the following disclaimer.
+* 2. Redistributions in binary form must reproduce the above copyright
+*    notice, this list of conditions and the following disclaimer in the
+*    documentation and/or other materials provided with the distribution.
+* 3. All advertising materials mentioning features or use of this software
+*    must display the following acknowledgement:
+*    This product includes software developed by the Delft University of Technology.
+* 4. Neither the name of the Delft University of Technology nor the names of
+*    its contributors may be used to endorse or promote products derived from
+*    this software without specific prior written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY LAURENS VAN DER MAATEN ''AS IS'' AND ANY EXPRESS
+* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+* EVENT SHALL LAURENS VAN DER MAATEN BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+* OF SUCH DAMAGE.
+*
+*/
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Collections;
+using HeuristicLab.Random;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Vantage point tree  (or VP tree) is a metric tree that segregates data in a metric space by choosing 
+  /// a position in the space (the "vantage point") and partitioning the data points into two parts: 
+  /// those points that are nearer to the vantage point than a threshold, and those points that are not. 
+  /// By recursively applying this procedure to partition the data into smaller and smaller sets, a tree 
+  /// data structure is created where neighbors in the tree are likely to be neighbors in the space.
+  /// </summary>
+  /// <typeparam name="T"></typeparam>
+  public class VantagePointTree<T> {
+    private readonly List<T> items;
+    private readonly Node root;
+    private readonly IDistance<T> distance;
+
+    public VantagePointTree(IDistance<T> distance, IEnumerable<T> items) : base() {
+      root = null;
+      this.distance = distance;
+      this.items = items.Select(x => x).ToList();
+      root = BuildFromPoints(0, this.items.Count);
+    }
+
+    /// <summary>
+    /// provides the k-nearest neighbours to a certain target element
+    /// </summary>
+    /// <param name="target">The target element</param>
+    /// <param name="k">How many neighbours</param>
+    /// <param name="results">The nearest neighbouring elements</param>
+    /// <param name="distances">The distances form the target corresponding to the neighbouring elements</param>
+    public void Search(T target, int k, out IList<T> results, out IList<double> distances) {
+      var heap = new PriorityQueue<double, IndexedItem<double>>(double.MaxValue, double.MinValue, k);
+      var tau = double.MaxValue;
+      Search(root, target, k, heap, ref tau);
+      var res = new List<T>();
+      var dist = new List<double>();
+      while (heap.Size > 0) {
+        res.Add(items[heap.PeekMinValue().Index]);// actually max distance
+        dist.Add(heap.PeekMinValue().Value);
+        heap.RemoveMin();
+      }
+      res.Reverse();
+      dist.Reverse();
+      results = res;
+      distances = dist;
+    }
+
+    private void Search(Node node, T target, int k, PriorityQueue<double, IndexedItem<double>> heap, ref double tau) {
+      if (node == null) return;
+      var dist = distance.Get(items[node.index], target);
+      if (dist < tau) {
+        if (heap.Size == k) heap.RemoveMin();   // remove furthest node from result list (if we already have k results) 
+        heap.Insert(-dist, new IndexedItem<double>(node.index, dist));
+        if (heap.Size == k) tau = heap.PeekMinValue().Value;
+      }
+      if (node.left == null && node.right == null) return;
+
+      if (dist < node.threshold) {
+        if (dist - tau <= node.threshold) Search(node.left, target, k, heap, ref tau);   // if there can still be neighbors inside the ball, recursively search left child first
+        if (dist + tau >= node.threshold) Search(node.right, target, k, heap, ref tau);  // if there can still be neighbors outside the ball, recursively search right child
+      } else {
+        if (dist + tau >= node.threshold) Search(node.right, target, k, heap, ref tau);  // if there can still be neighbors outside the ball, recursively search right child first
+        if (dist - tau <= node.threshold) Search(node.left, target, k, heap, ref tau);   // if there can still be neighbors inside the ball, recursively search left child
+      }
+    }
+
+    private Node BuildFromPoints(int lower, int upper) {
+      if (upper == lower)      // indicates that we're done here!
+        return null;
+
+      // Lower index is center of current node
+      var node = new Node { index = lower };
+      var r = new MersenneTwister(); //outer behaviour does not change with the random seed => no need to take the IRandom from the algorithm 
+      if (upper - lower <= 1) return node;
+
+      // if we did not arrive at leaf yet
+      // Choose an arbitrary point and move it to the start
+      var i = (int)(r.NextDouble() * (upper - lower - 1)) + lower;
+      items.Swap(lower, i);
+
+      // Partition around the median distance
+      var median = (upper + lower) / 2;
+      items.PartialSort(lower + 1, upper - 1, median, distance.GetDistanceComparer(items[lower]));
+
+      // Threshold of the new node will be the distance to the median
+      node.threshold = distance.Get(items[lower], items[median]);
+
+      // Recursively build tree
+      node.index = lower;
+      node.left = BuildFromPoints(lower + 1, median);
+      node.right = BuildFromPoints(median, upper);
+
+      // Return result
+      return node;
+    }
+
+    private sealed class Node {
+      public int index;
+      public double threshold;
+      public Node left;
+      public Node right;
+
+      internal Node() {
+        index = 0;
+        threshold = 0;
+        left = null;
+        right = null;
+      }
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TimeSeries/AutoregressiveModeling.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TimeSeries/AutoregressiveModeling.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/TimeSeries/AutoregressiveModeling.cs	(revision 17041)
@@ -0,0 +1,129 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.TimeSeriesPrognosis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.TimeSeries {
+  [Item("Autoregressive Modeling (AR)", "Timeseries modeling algorithm that creates AR-N models.")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 130)]
+  [StorableType("3BB9D723-944A-420F-A44A-A86F6FB4DABE")]
+  public class AutoregressiveModeling : FixedDataAnalysisAlgorithm<ITimeSeriesPrognosisProblem> {
+    private const string TimeOffesetParameterName = "Maximum Time Offset";
+
+    public IFixedValueParameter<IntValue> TimeOffsetParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[TimeOffesetParameterName]; }
+    }
+
+
+    public int TimeOffset {
+      get { return TimeOffsetParameter.Value.Value; }
+      set { TimeOffsetParameter.Value.Value = value; }
+    }
+
+    [StorableConstructor]
+    protected AutoregressiveModeling(StorableConstructorFlag _) : base(_) { }
+    protected AutoregressiveModeling(AutoregressiveModeling original, Cloner cloner) : base(original, cloner) { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new AutoregressiveModeling(this, cloner);
+    }
+
+    public AutoregressiveModeling()
+      : base() {
+      Parameters.Add(new FixedValueParameter<IntValue>(TimeOffesetParameterName, "The maximum time offset for the model ranging from 1 to infinity.", new IntValue(1)));
+      Problem = new TimeSeriesPrognosisProblem();
+    }
+
+    protected override void Run(CancellationToken cancellationToken) {
+      double rmsError, cvRmsError;
+      var solution = CreateAutoRegressiveSolution(Problem.ProblemData, TimeOffset, out rmsError, out cvRmsError);
+      Results.Add(new Result("Autoregressive solution", "The autoregressive time series prognosis solution.", solution));
+      Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the autoregressive time series prognosis solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Estimated root mean square error (cross-validation)", "The estimated root of the mean of squared errors of the autoregressive time series prognosis solution via cross validation.", new DoubleValue(cvRmsError)));
+    }
+
+    /// <summary>
+    /// Calculates an AR(p) model. For further information see http://en.wikipedia.org/wiki/Autoregressive_model
+    /// </summary>
+    /// <param name="problemData">The problem data which should be used for training</param>
+    /// <param name="timeOffset">The parameter p of the AR(p) specifying the maximum time offset [1,infinity] </param>
+    /// <returns>The times series autoregressive solution </returns>
+    public static ITimeSeriesPrognosisSolution CreateAutoRegressiveSolution(ITimeSeriesPrognosisProblemData problemData, int timeOffset) {
+      double rmsError, cvRmsError;
+      return CreateAutoRegressiveSolution(problemData, timeOffset, out rmsError, out cvRmsError);
+    }
+
+    private static ITimeSeriesPrognosisSolution CreateAutoRegressiveSolution(ITimeSeriesPrognosisProblemData problemData, int timeOffset, out double rmsError, out double cvRmsError) {
+      string targetVariable = problemData.TargetVariable;
+
+      double[,] inputMatrix = new double[problemData.TrainingPartition.Size, timeOffset + 1];
+      var targetValues = problemData.Dataset.GetDoubleValues(targetVariable).ToList();
+      for (int i = 0, row = problemData.TrainingPartition.Start; i < problemData.TrainingPartition.Size; i++, row++) {
+        for (int col = 0; col < timeOffset; col++) {
+          inputMatrix[i, col] = targetValues[row - col - 1];
+        }
+      }
+      // set target values in last column
+      for (int i = 0; i < inputMatrix.GetLength(0); i++)
+        inputMatrix[i, timeOffset] = targetValues[i + problemData.TrainingPartition.Start];
+
+      if (inputMatrix.ContainsNanOrInfinity())
+        throw new NotSupportedException("Linear regression does not support NaN or infinity values in the input dataset.");
+
+
+      alglib.linearmodel lm = new alglib.linearmodel();
+      alglib.lrreport ar = new alglib.lrreport();
+      int nRows = inputMatrix.GetLength(0);
+      int nFeatures = inputMatrix.GetLength(1) - 1;
+      double[] coefficients = new double[nFeatures + 1]; // last coefficient is for the constant
+
+      int retVal = 1;
+      alglib.lrbuild(inputMatrix, nRows, nFeatures, out retVal, out lm, out ar);
+      if (retVal != 1) throw new ArgumentException("Error in calculation of linear regression solution");
+      rmsError = ar.rmserror;
+      cvRmsError = ar.cvrmserror;
+
+      alglib.lrunpack(lm, out coefficients, out nFeatures);
+
+      var tree = LinearModelToTreeConverter.CreateTree(
+        variableNames: Enumerable.Repeat(problemData.TargetVariable, nFeatures).ToArray(),
+        lags: Enumerable.Range(0, timeOffset).Select(i => (i + 1) * -1).ToArray(),
+        coefficients: coefficients.Take(nFeatures).ToArray(),
+        @const: coefficients[nFeatures]
+        );
+
+      var interpreter = new SymbolicTimeSeriesPrognosisExpressionTreeInterpreter(problemData.TargetVariable);
+      var model = new SymbolicTimeSeriesPrognosisModel(problemData.TargetVariable, tree, interpreter);
+      var solution = model.CreateTimeSeriesPrognosisSolution((ITimeSeriesPrognosisProblemData)problemData.Clone());
+      return solution;
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClustering.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClustering.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClustering.cs	(revision 17041)
@@ -0,0 +1,104 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// k-Means clustering algorithm data analysis algorithm.
+  /// </summary>
+  [Item("k-Means", "The k-Means clustering algorithm (wrapper for ALGLIB).")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 10)]
+  [StorableType("84FAF4ED-0679-40BA-B3DD-83809095F8CF")]
+  public sealed class KMeansClustering : FixedDataAnalysisAlgorithm<IClusteringProblem> {
+    private const string KParameterName = "k";
+    private const string RestartsParameterName = "Restarts";
+    private const string KMeansSolutionResultName = "k-Means clustering solution";
+    #region parameter properties
+    public IValueParameter<IntValue> KParameter {
+      get { return (IValueParameter<IntValue>)Parameters[KParameterName]; }
+    }
+    public IValueParameter<IntValue> RestartsParameter {
+      get { return (IValueParameter<IntValue>)Parameters[RestartsParameterName]; }
+    }
+    #endregion
+    #region properties
+    public IntValue K {
+      get { return KParameter.Value; }
+    }
+    public IntValue Restarts {
+      get { return RestartsParameter.Value; }
+    }
+    #endregion
+    [StorableConstructor]
+    private KMeansClustering(StorableConstructorFlag _) : base(_) { }
+    private KMeansClustering(KMeansClustering original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public KMeansClustering()
+      : base() {
+      Parameters.Add(new ValueParameter<IntValue>(KParameterName, "The number of clusters.", new IntValue(3)));
+      Parameters.Add(new ValueParameter<IntValue>(RestartsParameterName, "The number of restarts.", new IntValue(0)));
+      Problem = new ClusteringProblem();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new KMeansClustering(this, cloner);
+    }
+
+    #region k-Means clustering
+    protected override void Run(CancellationToken cancellationToken) {
+      var solution = CreateKMeansSolution(Problem.ProblemData, K.Value, Restarts.Value);
+      Results.Add(new Result(KMeansSolutionResultName, "The k-Means clustering solution.", solution));
+    }
+
+    public static KMeansClusteringSolution CreateKMeansSolution(IClusteringProblemData problemData, int k, int restarts) {
+      var dataset = problemData.Dataset;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+      int info;
+      double[,] centers;
+      int[] xyc;
+      double[,] inputMatrix = dataset.ToArray(allowedInputVariables, rows);
+      if (inputMatrix.ContainsNanOrInfinity())
+        throw new NotSupportedException("k-Means clustering does not support NaN or infinity values in the input dataset.");
+
+      alglib.kmeansgenerate(inputMatrix, inputMatrix.GetLength(0), inputMatrix.GetLength(1), k, restarts + 1, out info, out centers, out xyc);
+      if (info != 1) throw new ArgumentException("Error in calculation of k-Means clustering solution");
+
+      KMeansClusteringSolution solution = new KMeansClusteringSolution(new KMeansClusteringModel(centers, allowedInputVariables), (IClusteringProblemData)problemData.Clone());
+      return solution;
+    }
+    #endregion
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringModel.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringModel.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringModel.cs	(revision 17041)
@@ -0,0 +1,97 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a k-Means clustering model.
+  /// </summary>
+  [StorableType("61D987AC-A142-433B-901C-B124E12A1C55")]
+  [Item("KMeansClusteringModel", "Represents a k-Means clustering model.")]
+  public sealed class KMeansClusteringModel : DataAnalysisModel, IClusteringModel {
+    public static new Image StaticItemImage {
+      get { return HeuristicLab.Common.Resources.VSImageLibrary.Function; }
+    }
+
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return allowedInputVariables; }
+    }
+
+    [Storable]
+    private string[] allowedInputVariables;
+    public IEnumerable<string> AllowedInputVariables {
+      get { return allowedInputVariables; }
+    }
+    [Storable]
+    private List<double[]> centers;
+    public IEnumerable<double[]> Centers {
+      get {
+        return centers.Select(x => (double[])x.Clone());
+      }
+    }
+    [StorableConstructor]
+    private KMeansClusteringModel(StorableConstructorFlag _) : base(_) { }
+    private KMeansClusteringModel(KMeansClusteringModel original, Cloner cloner)
+      : base(original, cloner) {
+      this.allowedInputVariables = (string[])original.allowedInputVariables.Clone();
+      this.centers = new List<double[]>(original.Centers);
+    }
+    public KMeansClusteringModel(double[,] centers, IEnumerable<string> allowedInputVariables)
+      : base() {
+      this.name = ItemName;
+      this.description = ItemDescription;
+      // disect center matrix into list of double[]
+      // centers are given as double matrix where number of rows = dimensions and number of columns = clusters
+      // each column is a cluster center
+      this.centers = new List<double[]>();
+      for (int i = 0; i < centers.GetLength(1); i++) {
+        double[] c = new double[centers.GetLength(0)];
+        for (int j = 0; j < c.Length; j++) {
+          c[j] = centers[j, i];
+        }
+        this.centers.Add(c);
+      }
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new KMeansClusteringModel(this, cloner);
+    }
+
+    public override bool IsProblemDataCompatible(IDataAnalysisProblemData problemData, out string errorMessage) {
+      if (problemData == null) throw new ArgumentNullException("problemData", "The provided problemData is null.");
+      return IsDatasetCompatible(problemData.Dataset, out errorMessage);
+    }
+
+
+    public IEnumerable<int> GetClusterValues(IDataset dataset, IEnumerable<int> rows) {
+      return KMeansClusteringUtil.FindClosestCenters(centers, dataset, allowedInputVariables, rows);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringSolution.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringSolution.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringSolution.cs	(revision 17041)
@@ -0,0 +1,60 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HEAL.Attic;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Optimization;
+using HeuristicLab.Data;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a k-Means clustering solution for a clustering problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("k-Means clustering solution", "Represents a k-Means solution for a clustering problem which can be visualized in the GUI.")]
+  [StorableType("E46AD906-04C4-4BEB-977D-BBC80E97C874")]
+  public sealed class KMeansClusteringSolution : ClusteringSolution {
+    private const string TrainingIntraClusterSumOfSquaresResultName = "Intra-cluster sum of squares (training)";
+    private const string TestIntraClusterSumOfSquaresResultName = "Intra-cluster sum of squares (test)";
+    public new KMeansClusteringModel Model {
+      get { return (KMeansClusteringModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private KMeansClusteringSolution(StorableConstructorFlag _) : base(_) { }
+    private KMeansClusteringSolution(KMeansClusteringSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public KMeansClusteringSolution(KMeansClusteringModel model, IClusteringProblemData problemData)
+      : base(model, problemData) {
+      double trainingIntraClusterSumOfSquares = KMeansClusteringUtil.CalculateIntraClusterSumOfSquares(model, problemData.Dataset, problemData.TrainingIndices);
+      double testIntraClusterSumOfSquares = KMeansClusteringUtil.CalculateIntraClusterSumOfSquares(model, problemData.Dataset, problemData.TestIndices);
+      this.Add(new Result(TrainingIntraClusterSumOfSquaresResultName, "The sum of squared distances of points of the training partition to the cluster center (is minimized by k-Means).", new DoubleValue(trainingIntraClusterSumOfSquares)));
+      this.Add(new Result(TestIntraClusterSumOfSquaresResultName, "The sum of squared distances of points of the test partition to the cluster center (is minimized by k-Means).", new DoubleValue(testIntraClusterSumOfSquares)));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new KMeansClusteringSolution(this, cloner);
+    }
+  }
+}
Index: /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringUtil.cs
===================================================================
--- /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringUtil.cs	(revision 17041)
+++ /branches/2952_RF-ModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringUtil.cs	(revision 17041)
@@ -0,0 +1,111 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public static class KMeansClusteringUtil {
+    public static IEnumerable<int> FindClosestCenters(IEnumerable<double[]> centers, IDataset dataset, IEnumerable<string> allowedInputVariables, IEnumerable<int> rows) {
+      int nRows = rows.Count();
+      int nCols = allowedInputVariables.Count();
+      int[] closestCenter = new int[nRows];
+      double[] bestCenterDistance = Enumerable.Repeat(double.MaxValue, nRows).ToArray();
+      int centerIndex = 1;
+
+      foreach (double[] center in centers) {
+        if (nCols != center.Length) throw new ArgumentException();
+        int rowIndex = 0;
+        foreach (var row in rows) {
+          // calc euclidian distance of point to center
+          double centerDistance = 0;
+          int col = 0;
+          foreach (var inputVariable in allowedInputVariables) {
+            double d = center[col++] - dataset.GetDoubleValue(inputVariable, row);
+            d = d * d; // square;
+            centerDistance += d;
+            if (centerDistance > bestCenterDistance[rowIndex]) break;
+          }
+          if (centerDistance < bestCenterDistance[rowIndex]) {
+            bestCenterDistance[rowIndex] = centerDistance;
+            closestCenter[rowIndex] = centerIndex;
+          }
+          rowIndex++;
+        }
+        centerIndex++;
+      }
+      return closestCenter;
+    }
+
+    public static double CalculateIntraClusterSumOfSquares(KMeansClusteringModel model, IDataset dataset, IEnumerable<int> rows) {
+      List<int> clusterValues = model.GetClusterValues(dataset, rows).ToList();
+      List<string> allowedInputVariables = model.AllowedInputVariables.ToList();
+      int nCols = allowedInputVariables.Count;
+      Dictionary<int, List<double[]>> clusterPoints = new Dictionary<int, List<double[]>>();
+      Dictionary<int, double[]> clusterMeans = new Dictionary<int, double[]>();
+      foreach (var clusterValue in clusterValues.Distinct()) {
+        clusterPoints.Add(clusterValue, new List<double[]>());
+      }
+
+      // collect points of clusters
+      int clusterValueIndex = 0;
+      foreach (var row in rows) {
+        double[] p = new double[allowedInputVariables.Count];
+        for (int i = 0; i < nCols; i++) {
+          p[i] = dataset.GetDoubleValue(allowedInputVariables[i], row);
+        }
+        clusterPoints[clusterValues[clusterValueIndex++]].Add(p);
+      }
+      // calculate cluster means
+      foreach (var pair in clusterPoints) {
+        double[] mean = new double[nCols];
+        foreach (var p in pair.Value) {
+          for (int i = 0; i < nCols; i++) {
+            mean[i] += p[i];
+          }
+        }
+        for (int i = 0; i < nCols; i++) {
+          mean[i] /= pair.Value.Count;
+        }
+        clusterMeans[pair.Key] = mean;
+      }
+      // calculate distances
+      double allCenterDistances = 0;
+      foreach (var pair in clusterMeans) {
+        double[] mean = pair.Value;
+        double centerDistances = 0;
+        foreach (var clusterPoint in clusterPoints[pair.Key]) {
+          double centerDistance = 0;
+          for (int i = 0; i < nCols; i++) {
+            double d = mean[i] - clusterPoint[i];
+            d = d * d;
+            centerDistance += d;
+          }
+          centerDistances += centerDistance;
+        }
+        allCenterDistances += centerDistances;
+      }
+      return allCenterDistances;
+    }
+  }
+}
