Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -36,5 +36,5 @@
 namespace HeuristicLab.Algorithms.DataAnalysis {
   [Item("Cross Validation", "Cross-validation wrapper for data analysis algorithms.")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 100)]
   [StorableClass]
   public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent {
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/FixedDataAnalysisAlgorithm.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/FixedDataAnalysisAlgorithm.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/FixedDataAnalysisAlgorithm.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceConst.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceConst.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceConst.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinear.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinear.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinear.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinearArd.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinearArd.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinearArd.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMask.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMask.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMask.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMaternIso.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMaternIso.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMaternIso.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNeuralNetwork.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNeuralNetwork.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNeuralNetwork.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNoise.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNoise.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNoise.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePeriodic.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePeriodic.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePeriodic.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePiecewisePolynomial.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePiecewisePolynomial.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePiecewisePolynomial.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePolynomial.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePolynomial.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePolynomial.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceProduct.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceProduct.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceProduct.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticArd.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticArd.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticArd.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticIso.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticIso.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticIso.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceScale.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceScale.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceScale.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSpectralMixture.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSpectralMixture.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSpectralMixture.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialArd.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialArd.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialArd.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialIso.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialIso.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialIso.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSum.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSum.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSum.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessBase.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessBase.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessBase.cs	(revision 13085)
@@ -2,5 +2,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -108,4 +108,8 @@
       Parameters[ApproximateGradientsParameterName].Hidden = true; // should not be changed
 
+      // necessary for BFGS
+      Parameters.Add(new ValueParameter<BoolValue>("Maximization", new BoolValue(false)));
+      Parameters["Maximization"].Hidden = true;
+
       var randomCreator = new HeuristicLab.Random.RandomCreator();
       var gpInitializer = new GaussianProcessHyperparameterInitializer();
@@ -181,4 +185,11 @@
     [StorableHook(HookType.AfterDeserialization)]
     private void AfterDeserialization() {
+      // BackwardsCompatibility3.4
+      #region Backwards compatible code, remove with 3.5
+      if (!Parameters.ContainsKey("Maximization")) {
+        Parameters.Add(new ValueParameter<BoolValue>("Maximization", new BoolValue(false)));
+        Parameters["Maximization"].Hidden = true;
+      }
+      #endregion
     }
   }
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassification.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassification.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassification.cs	(revision 13085)
@@ -2,5 +2,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -23,9 +23,6 @@
 using System;
 using System.Linq;
-using HeuristicLab.Algorithms.GradientDescent;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
-using HeuristicLab.Data;
-using HeuristicLab.Operators;
 using HeuristicLab.Optimization;
 using HeuristicLab.Parameters;
@@ -39,5 +36,5 @@
   /// </summary>
   [Item("Gaussian Process Least-Squares Classification", "Gaussian process least-squares classification data analysis algorithm.")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 160)]
   [StorableClass]
   public sealed class GaussianProcessClassification : GaussianProcessBase, IStorableContent {
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationModelCreator.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationModelCreator.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationModelCreator.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationSolutionCreator.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationSolutionCreator.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationSolutionCreator.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessCovarianceOptimizationProblem.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessCovarianceOptimizationProblem.cs	(revision 13085)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessCovarianceOptimizationProblem.cs	(revision 13085)
@@ -0,0 +1,341 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.Instances;
+
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("Gaussian Process Covariance Optimization Problem", "")]
+  [Creatable(CreatableAttribute.Categories.GeneticProgrammingProblems, Priority = 300)]
+  [StorableClass]
+  public sealed class GaussianProcessCovarianceOptimizationProblem : SymbolicExpressionTreeProblem, IRegressionProblem, IProblemInstanceConsumer<IRegressionProblemData>, IProblemInstanceExporter<IRegressionProblemData> {
+    #region static variables and ctor
+    private static readonly CovarianceMaternIso maternIso1;
+    private static readonly CovarianceMaternIso maternIso3;
+    private static readonly CovarianceMaternIso maternIso5;
+    private static readonly CovariancePiecewisePolynomial piecewisePoly0;
+    private static readonly CovariancePiecewisePolynomial piecewisePoly1;
+    private static readonly CovariancePiecewisePolynomial piecewisePoly2;
+    private static readonly CovariancePiecewisePolynomial piecewisePoly3;
+    private static readonly CovariancePolynomial poly2;
+    private static readonly CovariancePolynomial poly3;
+    private static readonly CovarianceSpectralMixture spectralMixture1;
+    private static readonly CovarianceSpectralMixture spectralMixture3;
+    private static readonly CovarianceSpectralMixture spectralMixture5;
+    private static readonly CovarianceLinear linear;
+    private static readonly CovarianceLinearArd linearArd;
+    private static readonly CovarianceNeuralNetwork neuralNetwork;
+    private static readonly CovariancePeriodic periodic;
+    private static readonly CovarianceRationalQuadraticIso ratQuadraticIso;
+    private static readonly CovarianceRationalQuadraticArd ratQuadraticArd;
+    private static readonly CovarianceSquaredExponentialArd sqrExpArd;
+    private static readonly CovarianceSquaredExponentialIso sqrExpIso;
+
+    static GaussianProcessCovarianceOptimizationProblem() {
+      // cumbersome initialization because of ConstrainedValueParameters
+      maternIso1 = new CovarianceMaternIso(); SetConstrainedValueParameter(maternIso1.DParameter, 1);
+      maternIso3 = new CovarianceMaternIso(); SetConstrainedValueParameter(maternIso3.DParameter, 3);
+      maternIso5 = new CovarianceMaternIso(); SetConstrainedValueParameter(maternIso5.DParameter, 5);
+
+      piecewisePoly0 = new CovariancePiecewisePolynomial(); SetConstrainedValueParameter(piecewisePoly0.VParameter, 0);
+      piecewisePoly1 = new CovariancePiecewisePolynomial(); SetConstrainedValueParameter(piecewisePoly1.VParameter, 1);
+      piecewisePoly2 = new CovariancePiecewisePolynomial(); SetConstrainedValueParameter(piecewisePoly2.VParameter, 2);
+      piecewisePoly3 = new CovariancePiecewisePolynomial(); SetConstrainedValueParameter(piecewisePoly3.VParameter, 3);
+
+      poly2 = new CovariancePolynomial(); poly2.DegreeParameter.Value.Value = 2;
+      poly3 = new CovariancePolynomial(); poly3.DegreeParameter.Value.Value = 3;
+
+      spectralMixture1 = new CovarianceSpectralMixture(); spectralMixture1.QParameter.Value.Value = 1;
+      spectralMixture3 = new CovarianceSpectralMixture(); spectralMixture3.QParameter.Value.Value = 3;
+      spectralMixture5 = new CovarianceSpectralMixture(); spectralMixture5.QParameter.Value.Value = 5;
+
+      linear = new CovarianceLinear();
+      linearArd = new CovarianceLinearArd();
+      neuralNetwork = new CovarianceNeuralNetwork();
+      periodic = new CovariancePeriodic();
+      ratQuadraticArd = new CovarianceRationalQuadraticArd();
+      ratQuadraticIso = new CovarianceRationalQuadraticIso();
+      sqrExpArd = new CovarianceSquaredExponentialArd();
+      sqrExpIso = new CovarianceSquaredExponentialIso();
+    }
+
+    private static void SetConstrainedValueParameter(IConstrainedValueParameter<IntValue> param, int val) {
+      param.Value = param.ValidValues.Single(v => v.Value == val);
+    }
+
+    #endregion
+
+    #region parameter names
+
+    private const string ProblemDataParameterName = "ProblemData";
+    private const string ConstantOptIterationsParameterName = "Constant optimization steps";
+    private const string RestartsParameterName = "Restarts";
+
+    #endregion
+
+    #region Parameter Properties
+    IParameter IDataAnalysisProblem.ProblemDataParameter { get { return ProblemDataParameter; } }
+
+    public IValueParameter<IRegressionProblemData> ProblemDataParameter {
+      get { return (IValueParameter<IRegressionProblemData>)Parameters[ProblemDataParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> ConstantOptIterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[ConstantOptIterationsParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> RestartsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[RestartsParameterName]; }
+    }
+    #endregion
+
+    #region Properties
+
+    public IRegressionProblemData ProblemData {
+      get { return ProblemDataParameter.Value; }
+      set { ProblemDataParameter.Value = value; }
+    }
+    IDataAnalysisProblemData IDataAnalysisProblem.ProblemData { get { return ProblemData; } }
+
+    public int ConstantOptIterations {
+      get { return ConstantOptIterationsParameter.Value.Value; }
+      set { ConstantOptIterationsParameter.Value.Value = value; }
+    }
+
+    public int Restarts {
+      get { return RestartsParameter.Value.Value; }
+      set { RestartsParameter.Value.Value = value; }
+    }
+    #endregion
+
+    public override bool Maximization {
+      get { return true; } // return log likelihood (instead of negative log likelihood as in GPR
+    }
+
+    public GaussianProcessCovarianceOptimizationProblem()
+      : base() {
+      Parameters.Add(new ValueParameter<IRegressionProblemData>(ProblemDataParameterName, "The data for the regression problem", new RegressionProblemData()));
+      Parameters.Add(new FixedValueParameter<IntValue>(ConstantOptIterationsParameterName, "Number of optimization steps for hyperparameter values", new IntValue(50)));
+      Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of random restarts for constant optimization.", new IntValue(10)));
+      Parameters["Restarts"].Hidden = true;
+      var g = new SimpleSymbolicExpressionGrammar();
+      g.AddSymbols(new string[] { "Sum", "Product" }, 2, 2);
+      g.AddTerminalSymbols(new string[]
+      {
+        "Linear",
+        "LinearArd",
+        "MaternIso1",
+        "MaternIso3",
+        "MaternIso5",
+        "NeuralNetwork",
+        "Periodic",
+        "PiecewisePolynomial0",
+        "PiecewisePolynomial1",
+        "PiecewisePolynomial2",
+        "PiecewisePolynomial3",
+        "Polynomial2",
+        "Polynomial3",
+        "RationalQuadraticArd",
+        "RationalQuadraticIso",
+        "SpectralMixture1",
+        "SpectralMixture3",
+        "SpectralMixture5",
+        "SquaredExponentialArd",
+        "SquaredExponentialIso"
+      });
+      base.Encoding = new SymbolicExpressionTreeEncoding(g, 10, 5);
+    }
+
+
+    public override double Evaluate(ISymbolicExpressionTree tree, IRandom random) {
+
+      var meanFunction = new MeanConst();
+      var problemData = ProblemData;
+      var ds = problemData.Dataset;
+      var targetVariable = problemData.TargetVariable;
+      var allowedInputVariables = problemData.AllowedInputVariables.ToArray();
+      var nVars = allowedInputVariables.Length;
+      var trainingRows = problemData.TrainingIndices.ToArray();
+
+      // use the same covariance function for each restart
+      var covarianceFunction = TreeToCovarianceFunction(tree);
+
+      // allocate hyperparameters
+      var hyperParameters = new double[meanFunction.GetNumberOfParameters(nVars) + covarianceFunction.GetNumberOfParameters(nVars) + 1]; // mean + cov + noise
+      double[] bestHyperParameters = new double[hyperParameters.Length];
+      var bestObjValue = new double[1] { double.MinValue };
+
+      // data that is necessary for the objective function
+      var data = Tuple.Create(ds, targetVariable, allowedInputVariables, trainingRows, (IMeanFunction)meanFunction, covarianceFunction, bestObjValue);
+
+      for (int t = 0; t < Restarts; t++) {
+        var prevBest = bestObjValue[0];
+        var prevBestHyperParameters = new double[hyperParameters.Length];
+        Array.Copy(bestHyperParameters, prevBestHyperParameters, bestHyperParameters.Length);
+
+        // initialize hyperparameters
+        hyperParameters[0] = ds.GetDoubleValues(targetVariable).Average(); // mean const
+
+        for (int i = 0; i < covarianceFunction.GetNumberOfParameters(nVars); i++) {
+          hyperParameters[1 + i] = random.NextDouble() * 2.0 - 1.0;
+        }
+        hyperParameters[hyperParameters.Length - 1] = 1.0; // s² = exp(2), TODO: other inits better?
+
+        // use alglib.bfgs for hyper-parameter optimization ...
+        double epsg = 0;
+        double epsf = 0.00001;
+        double epsx = 0;
+        double stpmax = 1;
+        int maxits = ConstantOptIterations;
+        alglib.mincgstate state;
+        alglib.mincgreport rep;
+
+        alglib.mincgcreate(hyperParameters, out state);
+        alglib.mincgsetcond(state, epsg, epsf, epsx, maxits);
+        alglib.mincgsetstpmax(state, stpmax);
+        alglib.mincgoptimize(state, ObjectiveFunction, null, data);
+
+        alglib.mincgresults(state, out bestHyperParameters, out rep);
+
+        if (rep.terminationtype < 0) {
+          // error -> restore previous best quality
+          bestObjValue[0] = prevBest;
+          Array.Copy(prevBestHyperParameters, bestHyperParameters, prevBestHyperParameters.Length);
+        }
+      }
+
+      return bestObjValue[0];
+    }
+
+    public void ObjectiveFunction(double[] x, ref double func, double[] grad, object obj) {
+      // we want to optimize the model likelihood by changing the hyperparameters and also return the gradient for each hyperparameter
+      var data = (Tuple<IDataset, string, string[], int[], IMeanFunction, ICovarianceFunction, double[]>)obj;
+      var ds = data.Item1;
+      var targetVariable = data.Item2;
+      var allowedInputVariables = data.Item3;
+      var trainingRows = data.Item4;
+      var meanFunction = data.Item5;
+      var covarianceFunction = data.Item6;
+      var bestObjValue = data.Item7;
+      var hyperParameters = x; // the decision variable vector
+
+      try {
+        var model = new GaussianProcessModel(ds, targetVariable, allowedInputVariables, trainingRows, hyperParameters, meanFunction, covarianceFunction);
+
+        func = model.NegativeLogLikelihood; // mincgoptimize, so we return negative likelihood
+        bestObjValue[0] = Math.Max(bestObjValue[0], -func); // problem itself is a maximization problem
+        var gradients = model.HyperparameterGradients;
+        Array.Copy(gradients, grad, gradients.Length);
+      } catch (Exception) {
+        // building the GaussianProcessModel might fail, in this case we return the worst possible objective value
+        func = 1.0E+300;
+        Array.Clear(grad, 0, grad.Length);
+      }
+    }
+
+    private ICovarianceFunction TreeToCovarianceFunction(ISymbolicExpressionTree tree) {
+      return TreeToCovarianceFunction(tree.Root.GetSubtree(0).GetSubtree(0)); // skip programroot and startsymbol
+    }
+
+    private ICovarianceFunction TreeToCovarianceFunction(ISymbolicExpressionTreeNode node) {
+      switch (node.Symbol.Name) {
+        case "Sum": {
+            var sum = new CovarianceSum();
+            sum.Terms.Add(TreeToCovarianceFunction(node.GetSubtree(0)));
+            sum.Terms.Add(TreeToCovarianceFunction(node.GetSubtree(1)));
+            return sum;
+          }
+        case "Product": {
+            var prod = new CovarianceProduct();
+            prod.Factors.Add(TreeToCovarianceFunction(node.GetSubtree(0)));
+            prod.Factors.Add(TreeToCovarianceFunction(node.GetSubtree(1)));
+            return prod;
+          }
+        // covFunction is cloned by the model so we can reuse instances of terminal covariance functions
+        case "Linear": return linear;
+        case "LinearArd": return linearArd;
+        case "MaternIso1": return maternIso1;
+        case "MaternIso3": return maternIso3;
+        case "MaternIso5": return maternIso5;
+        case "NeuralNetwork": return neuralNetwork;
+        case "Periodic": return periodic;
+        case "PiecewisePolynomial0": return piecewisePoly0;
+        case "PiecewisePolynomial1": return piecewisePoly1;
+        case "PiecewisePolynomial2": return piecewisePoly2;
+        case "PiecewisePolynomial3": return piecewisePoly3;
+        case "Polynomial2": return poly2;
+        case "Polynomial3": return poly3;
+        case "RationalQuadraticArd": return ratQuadraticArd;
+        case "RationalQuadraticIso": return ratQuadraticIso;
+        case "SpectralMixture1": return spectralMixture1;
+        case "SpectralMixture3": return spectralMixture3;
+        case "SpectralMixture5": return spectralMixture5;
+        case "SquaredExponentialArd": return sqrExpArd;
+        case "SquaredExponentialIso": return sqrExpIso;
+        default: throw new InvalidProgramException(string.Format("Found invalid symbol {0}", node.Symbol.Name));
+      }
+    }
+
+
+    // persistence
+    [StorableConstructor]
+    private GaussianProcessCovarianceOptimizationProblem(bool deserializing) : base(deserializing) { }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+    }
+
+    // cloning 
+    private GaussianProcessCovarianceOptimizationProblem(GaussianProcessCovarianceOptimizationProblem original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessCovarianceOptimizationProblem(this, cloner);
+    }
+
+    public void Load(IRegressionProblemData data) {
+      this.ProblemData = data;
+      OnProblemDataChanged();
+    }
+
+    public IRegressionProblemData Export() {
+      return ProblemData;
+    }
+
+    #region events
+    public event EventHandler ProblemDataChanged;
+
+
+    private void OnProblemDataChanged() {
+      var handler = ProblemDataChanged;
+      if (handler != null)
+        handler(this, EventArgs.Empty);
+    }
+    #endregion
+
+  }
+}
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessHyperparameterInitializer.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessHyperparameterInitializer.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessHyperparameterInitializer.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -85,9 +85,33 @@
     private double[] covarianceParameter;
 
-    [Storable]
-    private double[,] l;
-
-    [Storable]
-    private double[,] x;
+    private double[,] l; // used to be storable in previous versions (is calculated lazily now)
+    private double[,] x; // scaled training dataset, used to be storable in previous versions (is calculated lazily now)
+
+    // BackwardsCompatibility3.4
+    #region Backwards compatible code, remove with 3.5
+    [Storable(Name = "l")] // restore if available but don't store anymore
+    private double[,] l_storable {
+      set { this.l = value; }
+      get {
+        if (trainingDataset == null) return l; // this model has been created with an old version 
+        else return null; // if the training dataset is available l should not be serialized
+      }
+    }
+    [Storable(Name = "x")] // restore if available but don't store anymore
+    private double[,] x_storable {
+      set { this.x = value; }
+      get {
+        if (trainingDataset == null) return x; // this model has been created with an old version 
+        else return null; // if the training dataset is available x should not be serialized
+      }
+    }
+    #endregion
+
+
+    [Storable]
+    private IDataset trainingDataset; // it is better to store the original training dataset completely because this is more efficient in persistence
+    [Storable]
+    private int[] trainingRows;
+
     [Storable]
     private Scaling inputScaling;
@@ -101,4 +125,5 @@
       this.covarianceFunction = cloner.Clone(original.covarianceFunction);
       this.inputScaling = cloner.Clone(original.inputScaling);
+      this.trainingDataset = cloner.Clone(original.trainingDataset);
       this.negativeLogLikelihood = original.negativeLogLikelihood;
       this.targetVariable = original.targetVariable;
@@ -112,4 +137,5 @@
 
       // shallow copies of arrays because they cannot be modified
+      this.trainingRows = original.trainingRows;
       this.allowedInputVariables = original.allowedInputVariables;
       this.alpha = original.alpha;
@@ -117,5 +143,5 @@
       this.x = original.x;
     }
-    public GaussianProcessModel(Dataset ds, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<int> rows,
+    public GaussianProcessModel(IDataset ds, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<int> rows,
       IEnumerable<double> hyp, IMeanFunction meanFunction, ICovarianceFunction covarianceFunction)
       : base() {
@@ -137,17 +163,21 @@
                                              .ToArray();
       sqrSigmaNoise = Math.Exp(2.0 * hyp.Last());
-
       CalculateModel(ds, rows);
     }
 
-    private void CalculateModel(Dataset ds, IEnumerable<int> rows) {
-      inputScaling = new Scaling(ds, allowedInputVariables, rows);
-      x = AlglibUtil.PrepareAndScaleInputMatrix(ds, allowedInputVariables, rows, inputScaling);
+    private void CalculateModel(IDataset ds, IEnumerable<int> rows) {
+      this.trainingDataset = (IDataset)ds.Clone();
+      this.trainingRows = rows.ToArray();
+      this.inputScaling = new Scaling(trainingDataset, allowedInputVariables, rows);
+      this.x = CalculateX(trainingDataset, allowedInputVariables, rows, inputScaling);
       var y = ds.GetDoubleValues(targetVariable, rows);
 
       int n = x.GetLength(0);
-      l = new double[n, n];
-
-      // calculate means and covariances
+
+      // calculate cholesky decomposed (lower triangular) covariance matrix
+      var cov = covarianceFunction.GetParameterizedCovarianceFunction(covarianceParameter, Enumerable.Range(0, x.GetLength(1)));
+      this.l = CalculateL(x, cov, sqrSigmaNoise);
+
+      // calculate mean
       var mean = meanFunction.GetParameterizedMeanFunction(meanParameter, Enumerable.Range(0, x.GetLength(1)));
       double[] m = Enumerable.Range(0, x.GetLength(0))
@@ -155,25 +185,14 @@
         .ToArray();
 
-      var cov = covarianceFunction.GetParameterizedCovarianceFunction(covarianceParameter, Enumerable.Range(0, x.GetLength(1)));
-      for (int i = 0; i < n; i++) {
-        for (int j = i; j < n; j++) {
-          l[j, i] = cov.Covariance(x, i, j) / sqrSigmaNoise;
-          if (j == i) l[j, i] += 1.0;
-        }
-      }
-
-
-      // cholesky decomposition
+
+
+      // calculate sum of diagonal elements for likelihood
+      double diagSum = Enumerable.Range(0, n).Select(i => Math.Log(l[i, i])).Sum();
+
+      // solve for alpha
+      double[] ym = y.Zip(m, (a, b) => a - b).ToArray();
+
       int info;
       alglib.densesolverreport denseSolveRep;
-
-      var res = alglib.trfac.spdmatrixcholesky(ref l, n, false);
-      if (!res) throw new ArgumentException("Matrix is not positive semidefinite");
-
-      // calculate sum of diagonal elements for likelihood
-      double diagSum = Enumerable.Range(0, n).Select(i => Math.Log(l[i, i])).Sum();
-
-      // solve for alpha
-      double[] ym = y.Zip(m, (a, b) => a - b).ToArray();
 
       alglib.spdmatrixcholeskysolve(l, n, false, ym, out info, out denseSolveRep, out alpha);
@@ -230,4 +249,26 @@
     }
 
+    private static double[,] CalculateX(IDataset ds, IEnumerable<string> allowedInputVariables, IEnumerable<int> rows, Scaling inputScaling) {
+      return AlglibUtil.PrepareAndScaleInputMatrix(ds, allowedInputVariables, rows, inputScaling);
+    }
+
+    private static double[,] CalculateL(double[,] x, ParameterizedCovarianceFunction cov, double sqrSigmaNoise) {
+      int n = x.GetLength(0);
+      var l = new double[n, n];
+
+      // calculate covariances
+      for (int i = 0; i < n; i++) {
+        for (int j = i; j < n; j++) {
+          l[j, i] = cov.Covariance(x, i, j) / sqrSigmaNoise;
+          if (j == i) l[j, i] += 1.0;
+        }
+      }
+
+      // cholesky decomposition
+      var res = alglib.trfac.spdmatrixcholesky(ref l, n, false);
+      if (!res) throw new ArgumentException("Matrix is not positive semidefinite");
+      return l;
+    }
+
 
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -245,5 +286,5 @@
 
     #region IRegressionModel Members
-    public IEnumerable<double> GetEstimatedValues(Dataset dataset, IEnumerable<int> rows) {
+    public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
       return GetEstimatedValuesHelper(dataset, rows);
     }
@@ -257,8 +298,13 @@
 
 
-    private IEnumerable<double> GetEstimatedValuesHelper(Dataset dataset, IEnumerable<int> rows) {
+    private IEnumerable<double> GetEstimatedValuesHelper(IDataset dataset, IEnumerable<int> rows) {
+      if (x == null) {
+        this.x = CalculateX(trainingDataset, allowedInputVariables, trainingRows, inputScaling);
+      }
+      int n = x.GetLength(0);
+
       var newX = AlglibUtil.PrepareAndScaleInputMatrix(dataset, allowedInputVariables, rows, inputScaling);
       int newN = newX.GetLength(0);
-      int n = x.GetLength(0);
+
       var Ks = new double[newN, n];
       var mean = meanFunction.GetParameterizedMeanFunction(meanParameter, Enumerable.Range(0, newX.GetLength(1)));
@@ -277,12 +323,20 @@
     }
 
-    public IEnumerable<double> GetEstimatedVariance(Dataset dataset, IEnumerable<int> rows) {
+    public IEnumerable<double> GetEstimatedVariance(IDataset dataset, IEnumerable<int> rows) {
+      if (x == null) {
+        this.x = CalculateX(trainingDataset, allowedInputVariables, trainingRows, inputScaling);
+      }
+      int n = x.GetLength(0);
+
       var newX = AlglibUtil.PrepareAndScaleInputMatrix(dataset, allowedInputVariables, rows, inputScaling);
       int newN = newX.GetLength(0);
-      int n = x.GetLength(0);
 
       var kss = new double[newN];
       double[,] sWKs = new double[n, newN];
       var cov = covarianceFunction.GetParameterizedCovarianceFunction(covarianceParameter, Enumerable.Range(0, x.GetLength(1)));
+
+      if (l == null) {
+        l = CalculateL(x, cov, sqrSigmaNoise);
+      }
 
       // for stddev 
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModelCreator.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModelCreator.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModelCreator.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegression.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegression.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegression.cs	(revision 13085)
@@ -2,5 +2,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -23,9 +23,6 @@
 using System;
 using System.Linq;
-using HeuristicLab.Algorithms.GradientDescent;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
-using HeuristicLab.Data;
-using HeuristicLab.Operators;
 using HeuristicLab.Optimization;
 using HeuristicLab.Parameters;
@@ -39,5 +36,5 @@
   /// </summary>
   [Item("Gaussian Process Regression", "Gaussian process regression data analysis algorithm.")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 160)]
   [StorableClass]
   public sealed class GaussianProcessRegression : GaussianProcessBase, IStorableContent {
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/ICovarianceFunction.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/ICovarianceFunction.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/ICovarianceFunction.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/IMeanFunction.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/IMeanFunction.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/IMeanFunction.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanConst.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanConst.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanConst.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanLinear.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanLinear.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanLinear.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanProduct.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanProduct.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanProduct.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanSum.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanSum.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanSum.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanZero.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanZero.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanZero.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/Util.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/Util.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/Util.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithm.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithm.cs	(revision 13085)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithm.cs	(revision 13085)
@@ -0,0 +1,276 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Analysis;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.PluginInfrastructure;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("Gradient Boosted Trees", "Gradient boosted trees algorithm. Friedman, J. \"Greedy Function Approximation: A Gradient Boosting Machine\", IMS 1999 Reitz Lecture.")]
+  [StorableClass]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 125)]
+  public class GradientBoostedTreesAlgorithm : BasicAlgorithm {
+    public override Type ProblemType {
+      get { return typeof(IRegressionProblem); }
+    }
+    public new IRegressionProblem Problem {
+      get { return (IRegressionProblem)base.Problem; }
+      set { base.Problem = value; }
+    }
+
+    #region ParameterNames
+    private const string IterationsParameterName = "Iterations";
+    private const string MaxSizeParameterName = "Maximum Tree Size";
+    private const string NuParameterName = "Nu";
+    private const string RParameterName = "R";
+    private const string MParameterName = "M";
+    private const string SeedParameterName = "Seed";
+    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+    private const string LossFunctionParameterName = "LossFunction";
+    private const string UpdateIntervalParameterName = "UpdateInterval";
+    private const string CreateSolutionParameterName = "CreateSolution";
+    #endregion
+
+    #region ParameterProperties
+    public IFixedValueParameter<IntValue> IterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[IterationsParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> MaxSizeParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[MaxSizeParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> NuParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[NuParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> RParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[RParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> MParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[MParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> SeedParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+    public FixedValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (FixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+    public IConstrainedValueParameter<ILossFunction> LossFunctionParameter {
+      get { return (IConstrainedValueParameter<ILossFunction>)Parameters[LossFunctionParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> UpdateIntervalParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[UpdateIntervalParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> CreateSolutionParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
+    }
+    #endregion
+
+    #region Properties
+    public int Iterations {
+      get { return IterationsParameter.Value.Value; }
+      set { IterationsParameter.Value.Value = value; }
+    }
+    public int Seed {
+      get { return SeedParameter.Value.Value; }
+      set { SeedParameter.Value.Value = value; }
+    }
+    public bool SetSeedRandomly {
+      get { return SetSeedRandomlyParameter.Value.Value; }
+      set { SetSeedRandomlyParameter.Value.Value = value; }
+    }
+    public int MaxSize {
+      get { return MaxSizeParameter.Value.Value; }
+      set { MaxSizeParameter.Value.Value = value; }
+    }
+    public double Nu {
+      get { return NuParameter.Value.Value; }
+      set { NuParameter.Value.Value = value; }
+    }
+    public double R {
+      get { return RParameter.Value.Value; }
+      set { RParameter.Value.Value = value; }
+    }
+    public double M {
+      get { return MParameter.Value.Value; }
+      set { MParameter.Value.Value = value; }
+    }
+    public bool CreateSolution {
+      get { return CreateSolutionParameter.Value.Value; }
+      set { CreateSolutionParameter.Value.Value = value; }
+    }
+    #endregion
+
+    #region ResultsProperties
+    private double ResultsBestQuality {
+      get { return ((DoubleValue)Results["Best Quality"].Value).Value; }
+      set { ((DoubleValue)Results["Best Quality"].Value).Value = value; }
+    }
+    private DataTable ResultsQualities {
+      get { return ((DataTable)Results["Qualities"].Value); }
+    }
+    #endregion
+
+    [StorableConstructor]
+    protected GradientBoostedTreesAlgorithm(bool deserializing) : base(deserializing) { }
+
+    protected GradientBoostedTreesAlgorithm(GradientBoostedTreesAlgorithm original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GradientBoostedTreesAlgorithm(this, cloner);
+    }
+
+    public GradientBoostedTreesAlgorithm() {
+      Problem = new RegressionProblem(); // default problem
+
+      Parameters.Add(new FixedValueParameter<IntValue>(IterationsParameterName, "Number of iterations (set as high as possible, adjust in combination with nu, when increasing iterations also decrease nu)", new IntValue(1000)));
+      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<IntValue>(MaxSizeParameterName, "Maximal size of the tree learned in each step (prefer smaller sizes if possible)", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(RParameterName, "Ratio of training rows selected randomly in each step (0 < R <= 1)", new DoubleValue(0.5)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName, "Ratio of variables selected randomly in each step (0 < M <= 1)", new DoubleValue(0.5)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(NuParameterName, "Learning rate nu (step size for the gradient update, should be small 0 < nu < 0.1)", new DoubleValue(0.002)));
+      Parameters.Add(new FixedValueParameter<IntValue>(UpdateIntervalParameterName, "", new IntValue(100)));
+      Parameters[UpdateIntervalParameterName].Hidden = true;
+      Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+      Parameters[CreateSolutionParameterName].Hidden = true;
+
+      var lossFunctions = ApplicationManager.Manager.GetInstances<ILossFunction>();
+      Parameters.Add(new ConstrainedValueParameter<ILossFunction>(LossFunctionParameterName, "The loss function", new ItemSet<ILossFunction>(lossFunctions)));
+      LossFunctionParameter.Value = LossFunctionParameter.ValidValues.First(f => f.ToString().Contains("Squared")); // squared error loss is the default
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.4
+      #region Backwards compatible code, remove with 3.5
+      // parameter type has been changed
+      var lossFunctionParam = Parameters[LossFunctionParameterName] as ConstrainedValueParameter<StringValue>;
+      if (lossFunctionParam != null) {
+        Parameters.Remove(LossFunctionParameterName);
+        var selectedValue = lossFunctionParam.Value; // to be restored below
+
+        var lossFunctions = ApplicationManager.Manager.GetInstances<ILossFunction>();
+        Parameters.Add(new ConstrainedValueParameter<ILossFunction>(LossFunctionParameterName, "The loss function", new ItemSet<ILossFunction>(lossFunctions)));
+        // try to restore selected value
+        var selectedLossFunction =
+          LossFunctionParameter.ValidValues.FirstOrDefault(f => f.ToString() == selectedValue.Value);
+        if (selectedLossFunction != null) {
+          LossFunctionParameter.Value = selectedLossFunction;
+        } else {
+          LossFunctionParameter.Value = LossFunctionParameter.ValidValues.First(f => f.ToString().Contains("Squared")); // default: SE
+        }
+      }
+      #endregion
+    }
+
+    protected override void Run(CancellationToken cancellationToken) {
+      // Set up the algorithm
+      if (SetSeedRandomly) Seed = new System.Random().Next();
+
+      // Set up the results display
+      var iterations = new IntValue(0);
+      Results.Add(new Result("Iterations", iterations));
+
+      var table = new DataTable("Qualities");
+      table.Rows.Add(new DataRow("Loss (train)"));
+      table.Rows.Add(new DataRow("Loss (test)"));
+      Results.Add(new Result("Qualities", table));
+      var curLoss = new DoubleValue();
+      Results.Add(new Result("Loss (train)", curLoss));
+
+      // init
+      var problemData = (IRegressionProblemData)Problem.ProblemData.Clone();
+      var lossFunction = LossFunctionParameter.Value;
+      var state = GradientBoostedTreesAlgorithmStatic.CreateGbmState(problemData, lossFunction, (uint)Seed, MaxSize, R, M, Nu);
+
+      var updateInterval = UpdateIntervalParameter.Value.Value;
+      // Loop until iteration limit reached or canceled.
+      for (int i = 0; i < Iterations; i++) {
+        cancellationToken.ThrowIfCancellationRequested();
+
+        GradientBoostedTreesAlgorithmStatic.MakeStep(state);
+
+        // iteration results
+        if (i % updateInterval == 0) {
+          curLoss.Value = state.GetTrainLoss();
+          table.Rows["Loss (train)"].Values.Add(curLoss.Value);
+          table.Rows["Loss (test)"].Values.Add(state.GetTestLoss());
+          iterations.Value = i;
+        }
+      }
+
+      // final results
+      iterations.Value = Iterations;
+      curLoss.Value = state.GetTrainLoss();
+      table.Rows["Loss (train)"].Values.Add(curLoss.Value);
+      table.Rows["Loss (test)"].Values.Add(state.GetTestLoss());
+
+      // produce variable relevance
+      var orderedImpacts = state.GetVariableRelevance().Select(t => new { name = t.Key, impact = t.Value }).ToList();
+
+      var impacts = new DoubleMatrix();
+      var matrix = impacts as IStringConvertibleMatrix;
+      matrix.Rows = orderedImpacts.Count;
+      matrix.RowNames = orderedImpacts.Select(x => x.name);
+      matrix.Columns = 1;
+      matrix.ColumnNames = new string[] { "Relative variable relevance" };
+
+      int rowIdx = 0;
+      foreach (var p in orderedImpacts) {
+        matrix.SetValue(string.Format("{0:N2}", p.impact), rowIdx++, 0);
+      }
+
+      Results.Add(new Result("Variable relevance", impacts));
+      Results.Add(new Result("Loss (test)", new DoubleValue(state.GetTestLoss())));
+
+      // produce solution 
+      if (CreateSolution) {
+        var model = state.GetModel();
+
+        // for logistic regression we produce a classification solution
+        if (lossFunction is LogisticRegressionLoss) {
+          var classificationModel = new DiscriminantFunctionClassificationModel(model,
+            new AccuracyMaximizationThresholdCalculator());
+          var classificationProblemData = new ClassificationProblemData(problemData.Dataset,
+            problemData.AllowedInputVariables, problemData.TargetVariable, problemData.Transformations);
+          classificationModel.RecalculateModelParameters(classificationProblemData, classificationProblemData.TrainingIndices);
+
+          var classificationSolution = new DiscriminantFunctionClassificationSolution(classificationModel, classificationProblemData);
+          Results.Add(new Result("Solution", classificationSolution));
+        } else {
+          // otherwise we produce a regression solution
+          Results.Add(new Result("Solution", new RegressionSolution(model, problemData)));
+        }
+      }
+    }
+  }
+}
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithmStatic.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithmStatic.cs	(revision 13085)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithmStatic.cs	(revision 13085)
@@ -0,0 +1,201 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.Contracts;
+using System.Linq;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Random;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public static class GradientBoostedTreesAlgorithmStatic {
+    #region static API
+
+    public interface IGbmState {
+      IRegressionModel GetModel();
+      double GetTrainLoss();
+      double GetTestLoss();
+      IEnumerable<KeyValuePair<string, double>> GetVariableRelevance();
+    }
+
+    // created through factory method
+    // GbmState details are private API users can only use methods from IGbmState
+    private class GbmState : IGbmState {
+      internal IRegressionProblemData problemData { get; private set; }
+      internal ILossFunction lossFunction { get; private set; }
+      internal int maxSize { get; private set; }
+      internal double nu { get; private set; }
+      internal double r { get; private set; }
+      internal double m { get; private set; }
+      internal int[] trainingRows { get; private set; }
+      internal int[] testRows { get; private set; }
+      internal RegressionTreeBuilder treeBuilder { get; private set; }
+
+      private readonly uint randSeed;
+      private MersenneTwister random { get; set; }
+
+      // array members (allocate only once)
+      internal double[] pred;
+      internal double[] predTest;
+      internal double[] y;
+      internal int[] activeIdx;
+      internal double[] pseudoRes;
+
+      private readonly IList<IRegressionModel> models;
+      private readonly IList<double> weights;
+
+      public GbmState(IRegressionProblemData problemData, ILossFunction lossFunction, uint randSeed, int maxSize, double r, double m, double nu) {
+        // default settings for MaxSize, Nu and R
+        this.maxSize = maxSize;
+        this.nu = nu;
+        this.r = r;
+        this.m = m;
+
+        this.randSeed = randSeed;
+        random = new MersenneTwister(randSeed);
+        this.problemData = problemData;
+        this.trainingRows = problemData.TrainingIndices.ToArray();
+        this.testRows = problemData.TestIndices.ToArray();
+        this.lossFunction = lossFunction;
+
+        int nRows = trainingRows.Length;
+
+        y = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, trainingRows).ToArray();
+
+        treeBuilder = new RegressionTreeBuilder(problemData, random);
+
+        activeIdx = Enumerable.Range(0, nRows).ToArray();
+
+        var zeros = Enumerable.Repeat(0.0, nRows).ToArray();
+        double f0 = lossFunction.LineSearch(y, zeros, activeIdx, 0, nRows - 1); // initial constant value (mean for squared errors)
+        pred = Enumerable.Repeat(f0, nRows).ToArray();
+        predTest = Enumerable.Repeat(f0, testRows.Length).ToArray();
+        pseudoRes = new double[nRows];
+
+        models = new List<IRegressionModel>();
+        weights = new List<double>();
+        // add constant model
+        models.Add(new ConstantRegressionModel(f0));
+        weights.Add(1.0);
+      }
+
+      public IRegressionModel GetModel() {
+#pragma warning disable 618
+        var model = new GradientBoostedTreesModel(models, weights);
+#pragma warning restore 618
+        // we don't know the number of iterations here but the number of weights is equal 
+        // to the number of iterations + 1 (for the constant model)
+        // wrap the actual model in a surrogate that enables persistence and lazy recalculation of the model if necessary
+        return new GradientBoostedTreesModelSurrogate(problemData, randSeed, lossFunction, weights.Count - 1, maxSize, r, m, nu, model);
+      }
+      public IEnumerable<KeyValuePair<string, double>> GetVariableRelevance() {
+        return treeBuilder.GetVariableRelevance();
+      }
+
+      public double GetTrainLoss() {
+        int nRows = y.Length;
+        return lossFunction.GetLoss(y, pred) / nRows;
+      }
+      public double GetTestLoss() {
+        var yTest = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, testRows);
+        var nRows = testRows.Length;
+        return lossFunction.GetLoss(yTest, predTest) / nRows;
+      }
+
+      internal void AddModel(IRegressionModel m, double weight) {
+        models.Add(m);
+        weights.Add(weight);
+      }
+    }
+
+    // simple interface
+    public static IRegressionSolution TrainGbm(IRegressionProblemData problemData, ILossFunction lossFunction, int maxSize, double nu, double r, double m, int maxIterations, uint randSeed = 31415) {
+      Contract.Assert(r > 0);
+      Contract.Assert(r <= 1.0);
+      Contract.Assert(nu > 0);
+      Contract.Assert(nu <= 1.0);
+
+      var state = (GbmState)CreateGbmState(problemData, lossFunction, randSeed, maxSize, r, m, nu);
+
+      for (int iter = 0; iter < maxIterations; iter++) {
+        MakeStep(state);
+      }
+
+      var model = state.GetModel();
+      return new RegressionSolution(model, (IRegressionProblemData)problemData.Clone());
+    }
+
+    // for custom stepping & termination
+    public static IGbmState CreateGbmState(IRegressionProblemData problemData, ILossFunction lossFunction, uint randSeed, int maxSize = 3, double r = 0.66, double m = 0.5, double nu = 0.01) {
+      return new GbmState(problemData, lossFunction, randSeed, maxSize, r, m, nu);
+    }
+
+    // use default settings for maxSize, nu, r from state
+    public static void MakeStep(IGbmState state) {
+      var gbmState = state as GbmState;
+      if (gbmState == null) throw new ArgumentException("state");
+
+      MakeStep(gbmState, gbmState.maxSize, gbmState.nu, gbmState.r, gbmState.m);
+    }
+
+    // allow dynamic adaptation of maxSize, nu and r (even though this is not used)
+    public static void MakeStep(IGbmState state, int maxSize, double nu, double r, double m) {
+      var gbmState = state as GbmState;
+      if (gbmState == null) throw new ArgumentException("state");
+
+      var problemData = gbmState.problemData;
+      var lossFunction = gbmState.lossFunction;
+      var yPred = gbmState.pred;
+      var yPredTest = gbmState.predTest;
+      var treeBuilder = gbmState.treeBuilder;
+      var y = gbmState.y;
+      var activeIdx = gbmState.activeIdx;
+      var pseudoRes = gbmState.pseudoRes;
+      var trainingRows = gbmState.trainingRows;
+      var testRows = gbmState.testRows;
+
+      // copy output of gradient function to pre-allocated rim array (pseudo-residual per row and model)
+      int rimIdx = 0;
+      foreach (var g in lossFunction.GetLossGradient(y, yPred)) {
+        pseudoRes[rimIdx++] = g;
+      }
+
+      var tree = treeBuilder.CreateRegressionTreeForGradientBoosting(pseudoRes, yPred, maxSize, activeIdx, lossFunction, r, m);
+
+      int i = 0;
+      foreach (var pred in tree.GetEstimatedValues(problemData.Dataset, trainingRows)) {
+        yPred[i] = yPred[i] + nu * pred;
+        i++;
+      }
+      // update predictions for validation set
+      i = 0;
+      foreach (var pred in tree.GetEstimatedValues(problemData.Dataset, testRows)) {
+        yPredTest[i] = yPredTest[i] + nu * pred;
+        i++;
+      }
+
+      gbmState.AddModel(tree, nu);
+    }
+    #endregion
+  }
+}
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesModel.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesModel.cs	(revision 13085)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesModel.cs	(revision 13085)
@@ -0,0 +1,111 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item("Gradient boosted tree model", "")]
+  // this is essentially a collection of weighted regression models
+  public sealed class GradientBoostedTreesModel : NamedItem, IRegressionModel {
+    // BackwardsCompatibility3.4 for allowing deserialization & serialization of old models
+    #region Backwards compatible code, remove with 3.5
+    private bool isCompatibilityLoaded = false; // only set to true if the model is deserialized from the old format, needed to make sure that information is serialized again if it was loaded from the old format
+
+    [Storable(Name = "models")]
+    private IList<IRegressionModel> __persistedModels {
+      set {
+        this.isCompatibilityLoaded = true;
+        this.models.Clear();
+        foreach (var m in value) this.models.Add(m);
+      }
+      get { if (this.isCompatibilityLoaded) return models; else return null; }
+    }
+    [Storable(Name = "weights")]
+    private IList<double> __persistedWeights {
+      set {
+        this.isCompatibilityLoaded = true;
+        this.weights.Clear();
+        foreach (var w in value) this.weights.Add(w);
+      }
+      get { if (this.isCompatibilityLoaded) return weights; else return null; }
+    }
+    #endregion
+
+    private readonly IList<IRegressionModel> models;
+    public IEnumerable<IRegressionModel> Models { get { return models; } }
+
+    private readonly IList<double> weights;
+    public IEnumerable<double> Weights { get { return weights; } }
+
+    [StorableConstructor]
+    private GradientBoostedTreesModel(bool deserializing)
+      : base(deserializing) {
+      models = new List<IRegressionModel>();
+      weights = new List<double>();
+    }
+    private GradientBoostedTreesModel(GradientBoostedTreesModel original, Cloner cloner)
+      : base(original, cloner) {
+      this.weights = new List<double>(original.weights);
+      this.models = new List<IRegressionModel>(original.models.Select(m => cloner.Clone(m)));
+      this.isCompatibilityLoaded = original.isCompatibilityLoaded;
+    }
+    [Obsolete("The constructor of GBTModel should not be used directly anymore (use GBTModelSurrogate instead)")]
+    public GradientBoostedTreesModel(IEnumerable<IRegressionModel> models, IEnumerable<double> weights)
+      : base("Gradient boosted tree model", string.Empty) {
+      this.models = new List<IRegressionModel>(models);
+      this.weights = new List<double>(weights);
+
+      if (this.models.Count != this.weights.Count) throw new ArgumentException();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GradientBoostedTreesModel(this, cloner);
+    }
+
+    public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      // allocate target array go over all models and add up weighted estimation for each row
+      if (!rows.Any()) return Enumerable.Empty<double>(); // return immediately if rows is empty. This prevents multiple iteration over lazy rows enumerable.
+      // (which essentially looks up indexes in a dictionary)
+      var res = new double[rows.Count()];
+      for (int i = 0; i < models.Count; i++) {
+        var w = weights[i];
+        var m = models[i];
+        int r = 0;
+        foreach (var est in m.GetEstimatedValues(dataset, rows)) {
+          res[r++] += w * est;
+        }
+      }
+      return res;
+    }
+
+    public IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RegressionSolution(this, (IRegressionProblemData)problemData.Clone());
+    }
+  }
+}
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesModelSurrogate.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesModelSurrogate.cs	(revision 13085)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesModelSurrogate.cs	(revision 13085)
@@ -0,0 +1,113 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  // this class is used as a surrogate for persistence of an actual GBT model 
+  // since the actual GBT model would be very large when persisted we only store all necessary information to
+  // recalculate the actual GBT model on demand
+  [Item("Gradient boosted tree model", "")]
+  public sealed class GradientBoostedTreesModelSurrogate : NamedItem, IRegressionModel {
+    // don't store the actual model!
+    private IRegressionModel actualModel; // the actual model is only recalculated when necessary
+    public IRegressionModel Model { get { return actualModel; } }
+
+    [Storable]
+    private readonly IRegressionProblemData trainingProblemData;
+    [Storable]
+    private readonly uint seed;
+    [Storable]
+    private ILossFunction lossFunction;
+    [Storable]
+    private double r;
+    [Storable]
+    private double m;
+    [Storable]
+    private double nu;
+    [Storable]
+    private int iterations;
+    [Storable]
+    private int maxSize;
+
+
+    [StorableConstructor]
+    private GradientBoostedTreesModelSurrogate(bool deserializing) : base(deserializing) { }
+
+    private GradientBoostedTreesModelSurrogate(GradientBoostedTreesModelSurrogate original, Cloner cloner)
+      : base(original, cloner) {
+      if (original.actualModel != null) this.actualModel = cloner.Clone(original.actualModel);
+
+      this.trainingProblemData = cloner.Clone(original.trainingProblemData);
+      this.lossFunction = cloner.Clone(original.lossFunction);
+      this.seed = original.seed;
+      this.iterations = original.iterations;
+      this.maxSize = original.maxSize;
+      this.r = original.r;
+      this.m = original.m;
+      this.nu = original.nu;
+    }
+
+    // create only the surrogate model without an actual model
+    public GradientBoostedTreesModelSurrogate(IRegressionProblemData trainingProblemData, uint seed, ILossFunction lossFunction, int iterations, int maxSize, double r, double m, double nu)
+      : base("Gradient boosted tree model", string.Empty) {
+      this.trainingProblemData = trainingProblemData;
+      this.seed = seed;
+      this.lossFunction = lossFunction;
+      this.iterations = iterations;
+      this.maxSize = maxSize;
+      this.r = r;
+      this.m = m;
+      this.nu = nu;
+    }
+
+    // wrap an actual model in a surrograte
+    public GradientBoostedTreesModelSurrogate(IRegressionProblemData trainingProblemData, uint seed, ILossFunction lossFunction, int iterations, int maxSize, double r, double m, double nu, IRegressionModel model)
+      : this(trainingProblemData, seed, lossFunction, iterations, maxSize, r, m, nu) {
+      this.actualModel = model;
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GradientBoostedTreesModelSurrogate(this, cloner);
+    }
+
+    // forward message to actual model (recalculate model first if necessary)
+    public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      if (actualModel == null) actualModel = RecalculateModel();
+      return actualModel.GetEstimatedValues(dataset, rows);
+    }
+
+    public IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RegressionSolution(this, (IRegressionProblemData)problemData.Clone());
+    }
+
+
+    private IRegressionModel RecalculateModel() {
+      return GradientBoostedTreesAlgorithmStatic.TrainGbm(trainingProblemData, lossFunction, maxSize, nu, r, m, iterations, seed).Model;
+    }
+  }
+}
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/AbsoluteErrorLoss.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/AbsoluteErrorLoss.cs	(revision 13085)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/AbsoluteErrorLoss.cs	(revision 13085)
@@ -0,0 +1,94 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  // loss function for the weighted absolute error
+  [StorableClass]
+  [Item("Absolute error loss", "")]
+  public sealed class AbsoluteErrorLoss : Item, ILossFunction {
+    public AbsoluteErrorLoss() { }
+
+    public double GetLoss(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+
+      double s = 0;
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        double res = targetEnum.Current - predEnum.Current;
+        s += Math.Abs(res);  // |res|
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have differing lengths");
+
+      return s;
+    }
+
+    public IEnumerable<double> GetLossGradient(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        // dL(y, f(x)) / df(x) = sign(res)
+        var res = targetEnum.Current - predEnum.Current;
+        if (res > 0) yield return 1.0;
+        else if (res < 0) yield return -1.0;
+        else yield return 0.0;
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have differing lengths");
+    }
+
+    // return median of residuals
+    // targetArr and predArr are not changed by LineSearch
+    public double LineSearch(double[] targetArr, double[] predArr, int[] idx, int startIdx, int endIdx) {
+      if (targetArr.Length != predArr.Length)
+        throw new ArgumentException("target and pred have differing lengths");
+
+      // Median() is allocating an array anyway
+      // It would be possible to pre-allocated an array for the residuals if Median() would allow specification of a sub-range
+      int nRows = endIdx - startIdx + 1;
+      var res = new double[nRows];
+      for (int i = startIdx; i <= endIdx; i++) {
+        var row = idx[i];
+        res[i - startIdx] = targetArr[row] - predArr[row];
+      }
+      return res.Median(); // TODO: improve efficiency
+    }
+
+    #region item implementation
+    [StorableConstructor]
+    private AbsoluteErrorLoss(bool deserializing) : base(deserializing) { }
+
+    private AbsoluteErrorLoss(AbsoluteErrorLoss original, Cloner cloner) : base(original, cloner) { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new AbsoluteErrorLoss(this, cloner);
+    }
+    #endregion
+  }
+}
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/ILossFunction.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/ILossFunction.cs	(revision 13085)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/ILossFunction.cs	(revision 13085)
@@ -0,0 +1,42 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using HeuristicLab.Core;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  // represents an interface for loss functions used by gradient boosting
+  // target represents the target vector  (original targets from the problem data, never changed)
+  // pred   represents the current vector of predictions (a weighted combination of models learned so far, this vector is updated after each step)
+  public interface ILossFunction : IItem {
+    // returns the loss of the current prediction vector 
+    double GetLoss(IEnumerable<double> target, IEnumerable<double> pred);
+
+    // returns an enumerable of the loss gradient for each row
+    IEnumerable<double> GetLossGradient(IEnumerable<double> target, IEnumerable<double> pred);
+
+    // returns the optimal value for the partition of rows stored in idx[startIdx] .. idx[endIdx] inclusive
+    double LineSearch(double[] targetArr, double[] predArr, int[] idx, int startIdx, int endIdx);
+  }
+}
+
+
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/LogisticRegressionLoss.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/LogisticRegressionLoss.cs	(revision 13085)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/LogisticRegressionLoss.cs	(revision 13085)
@@ -0,0 +1,103 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  // Greedy Function Approximation: A Gradient Boosting Machine (page 9) 
+  [StorableClass]
+  [Item("Logistic regression loss", "")]
+  public sealed class LogisticRegressionLoss : Item, ILossFunction {
+    public LogisticRegressionLoss() { }
+
+    public double GetLoss(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+
+      double s = 0;
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        Debug.Assert(targetEnum.Current.IsAlmost(0.0) || targetEnum.Current.IsAlmost(1.0), "labels must be 0 or 1 for logistic regression loss");
+
+        var y = targetEnum.Current * 2 - 1; // y in {-1,1}
+        s += Math.Log(1 + Math.Exp(-2 * y * predEnum.Current));
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have different lengths");
+
+      return s;
+    }
+
+    public IEnumerable<double> GetLossGradient(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        Debug.Assert(targetEnum.Current.IsAlmost(0.0) || targetEnum.Current.IsAlmost(1.0), "labels must be 0 or 1 for logistic regression loss");
+        var y = targetEnum.Current * 2 - 1; // y in {-1,1}
+
+        yield return 2 * y / (1 + Math.Exp(2 * y * predEnum.Current));
+
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have different lengths");
+    }
+
+    // targetArr and predArr are not changed by LineSearch
+    public double LineSearch(double[] targetArr, double[] predArr, int[] idx, int startIdx, int endIdx) {
+      if (targetArr.Length != predArr.Length)
+        throw new ArgumentException("target and pred have different lengths");
+
+      // "Simple Newton-Raphson step" of eqn. 23 
+      double sumY = 0.0;
+      double sumDiff = 0.0;
+      for (int i = startIdx; i <= endIdx; i++) {
+        var row = idx[i];
+        var y = targetArr[row] * 2 - 1; // y in {-1,1}
+        var pseudoResponse = 2 * y / (1 + Math.Exp(2 * y * predArr[row]));
+
+        sumY += pseudoResponse;
+        sumDiff += Math.Abs(pseudoResponse) * (2 - Math.Abs(pseudoResponse));
+      }
+      // prevent divByZero
+      sumDiff = Math.Max(1E-12, sumDiff);
+      return sumY / sumDiff;
+    }
+
+    #region item implementation
+    [StorableConstructor]
+    private LogisticRegressionLoss(bool deserializing) : base(deserializing) { }
+
+    private LogisticRegressionLoss(LogisticRegressionLoss original, Cloner cloner) : base(original, cloner) { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new LogisticRegressionLoss(this, cloner);
+    }
+    #endregion
+
+  }
+}
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/QuantileRegressionLoss.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/QuantileRegressionLoss.cs	(revision 13085)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/QuantileRegressionLoss.cs	(revision 13085)
@@ -0,0 +1,118 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  // loss function for quantile regression
+  // Generalized Boosted Models - A Guide To The gbm Package, Greg Ridgeway, August 2007, page 11 
+  [StorableClass]
+  [Item("QuantileRegressionloss", "Loss function for quantile regression")]
+  public sealed class QuantileRegressionLoss : ParameterizedNamedItem, ILossFunction {
+    public IFixedValueParameter<PercentValue> AlphaParameter {
+      get { return (IFixedValueParameter<PercentValue>)Parameters["Alpha"]; }
+    }
+
+    public double Alpha {
+      get { return AlphaParameter.Value.Value; }
+      set {
+        if (value <= 0.0 || value >= 1.0) throw new ArgumentException("Valid values for alpha: 0 < alpha < 1");
+        AlphaParameter.Value.Value = value;
+      }
+    }
+
+    public QuantileRegressionLoss()
+      : base("QuantileRegressionLoss", "Loss function for quantile regression") {
+      Parameters.Add(new FixedValueParameter<PercentValue>("Alpha", new PercentValue(0.9)));
+    }
+
+    public double GetLoss(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+      var alpha = Alpha;
+      double leftSum = 0;
+      double rightsum = 0;
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        double res = targetEnum.Current - predEnum.Current;
+        if (res > 0) leftSum += res;
+        else rightsum += -res;
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have differing lengths");
+
+      return alpha * leftSum + (1 - alpha) * rightsum;
+    }
+
+    public IEnumerable<double> GetLossGradient(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+      var alpha = AlphaParameter.Value.Value;
+
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        var res = targetEnum.Current - predEnum.Current;
+        if (res > 0) yield return alpha;
+        else if (res < 0) yield return -(1.0 - alpha);
+        else yield return 0.0;
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have differing lengths");
+    }
+
+    // targetArr and predArr are not changed by LineSearch
+    public double LineSearch(double[] targetArr, double[] predArr, int[] idx, int startIdx, int endIdx) {
+      if (targetArr.Length != predArr.Length)
+        throw new ArgumentException("target and pred have differing lengths");
+
+      // Quantile() is allocating an array anyway
+      // It would be possible to pre-allocated an array for the residuals if Quantile() would allow specification of a sub-range
+      int nRows = endIdx - startIdx + 1;
+      var res = new double[nRows];
+      for (int i = startIdx; i <= endIdx; i++) {
+        var row = idx[i];
+        res[i - startIdx] = targetArr[row] - predArr[row];
+      }
+      return res.Quantile(Alpha);
+    }
+
+    #region item implementation
+    [StorableConstructor]
+    private QuantileRegressionLoss(bool deserializing) : base(deserializing) { }
+
+    private QuantileRegressionLoss(QuantileRegressionLoss original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new QuantileRegressionLoss(this, cloner);
+    }
+    #endregion
+
+    public override bool CanChangeName { get { return false; } }
+    public override bool CanChangeDescription { get { return false; } }
+  }
+}
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/RelativeErrorLoss.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/RelativeErrorLoss.cs	(revision 13085)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/RelativeErrorLoss.cs	(revision 13085)
@@ -0,0 +1,124 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  // relative error loss is a special case of weighted absolute error loss with weights = (1/target)
+  [StorableClass]
+  [Item("Relative error loss", "")]
+  public sealed class RelativeErrorLoss : Item, ILossFunction {
+    public RelativeErrorLoss() { }
+
+    public double GetLoss(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+
+      double s = 0;
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        double res = targetEnum.Current - predEnum.Current;
+        s += Math.Abs(res) * Math.Abs(1.0 / targetEnum.Current);
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have different lengths");
+
+      return s;
+    }
+
+    public IEnumerable<double> GetLossGradient(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        // sign(res) * abs(1 / target)
+        var res = targetEnum.Current - predEnum.Current;
+        if (res > 0) yield return 1.0 / Math.Abs(targetEnum.Current);
+        else if (res < 0) yield return -1.0 / Math.Abs(targetEnum.Current);
+        else yield return 0.0;
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have different lengths");
+    }
+
+    // targetArr and predArr are not changed by LineSearch
+    public double LineSearch(double[] targetArr, double[] predArr, int[] idx, int startIdx, int endIdx) {
+      if (targetArr.Length != predArr.Length)
+        throw new ArgumentException("target and pred have different lengths");
+
+      // line search for relative error
+      // weighted median (weight = 1/target)
+      int nRows = endIdx - startIdx + 1; // startIdx and endIdx are inclusive
+      if (nRows == 1) return targetArr[idx[startIdx]] - predArr[idx[startIdx]]; // res 
+      else if (nRows == 2) {
+        // weighted average of two residuals
+        var w0 = Math.Abs(1.0 / targetArr[idx[startIdx]]);
+        var w1 = Math.Abs(1.0 / targetArr[idx[endIdx]]);
+        if (w0 > w1) {
+          return targetArr[idx[startIdx]] - predArr[idx[startIdx]];
+        } else if (w0 < w1) {
+          return targetArr[idx[endIdx]] - predArr[idx[endIdx]];
+        } else {
+          // same weight -> return average of both residuals
+          return ((targetArr[idx[startIdx]] - predArr[idx[startIdx]]) + (targetArr[idx[endIdx]] - predArr[idx[endIdx]])) / 2;
+        }
+      } else {
+        // create an array of key-value pairs to be sorted (instead of using Array.Sort(res, weights))
+        var res_w = new KeyValuePair<double, double>[nRows];
+        var totalWeight = 0.0;
+        for (int i = startIdx; i <= endIdx; i++) {
+          int row = idx[i];
+          var res = targetArr[row] - predArr[row];
+          var w = Math.Abs(1.0 / targetArr[row]);
+          res_w[i - startIdx] = new KeyValuePair<double, double>(res, w);
+          totalWeight += w;
+        }
+        // TODO: improve efficiency (find median without sort)
+        res_w.StableSort((a, b) => Math.Sign(a.Key - b.Key));
+
+        int k = 0;
+        double sum = totalWeight - res_w[k].Value; // total - first weight
+        while (sum > totalWeight / 2) {
+          k++;
+          sum -= res_w[k].Value;
+        }
+        return res_w[k].Key;
+      }
+    }
+
+    #region item implementation
+    [StorableConstructor]
+    private RelativeErrorLoss(bool deserializing) : base(deserializing) { }
+
+    private RelativeErrorLoss(RelativeErrorLoss original, Cloner cloner) : base(original, cloner) { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RelativeErrorLoss(this, cloner);
+    }
+    #endregion
+  }
+}
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/SquaredErrorLoss.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/SquaredErrorLoss.cs	(revision 13085)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/SquaredErrorLoss.cs	(revision 13085)
@@ -0,0 +1,90 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item("Squared error loss", "")]
+  public sealed class SquaredErrorLoss : Item, ILossFunction {
+    public SquaredErrorLoss() { }
+
+    public double GetLoss(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+
+      double s = 0;
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        double res = targetEnum.Current - predEnum.Current;
+        s += res * res; // (res)^2
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have different lengths");
+
+      return s;
+    }
+
+    public IEnumerable<double> GetLossGradient(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        yield return 2.0 * (targetEnum.Current - predEnum.Current); // dL(y, f(x)) / df(x)  = 2 * res 
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have different lengths");
+    }
+
+    // targetArr and predArr are not changed by LineSearch
+    public double LineSearch(double[] targetArr, double[] predArr, int[] idx, int startIdx, int endIdx) {
+      if (targetArr.Length != predArr.Length)
+        throw new ArgumentException("target and pred have different lengths");
+
+      // line search for squared error loss 
+      // for a given partition of rows the optimal constant that should be added to the current prediction values is the average of the residuals
+      double s = 0.0;
+      int n = 0;
+      for (int i = startIdx; i <= endIdx; i++) {
+        int row = idx[i];
+        s += (targetArr[row] - predArr[row]);
+        n++;
+      }
+      return s / n;
+    }
+
+    #region item implementation
+    [StorableConstructor]
+    private SquaredErrorLoss(bool deserializing) : base(deserializing) { }
+
+    private SquaredErrorLoss(SquaredErrorLoss original, Cloner cloner) : base(original, cloner) { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new SquaredErrorLoss(this, cloner);
+    }
+    #endregion
+  }
+}
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeBuilder.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeBuilder.cs	(revision 13085)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeBuilder.cs	(revision 13085)
@@ -0,0 +1,474 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  // This class implements a greedy decision tree learner which selects splits with the maximum reduction in sum of squared errors.
+  // The tree builder also tracks variable relevance metrics based on the splits and improvement after the split.
+  // The implementation is tuned for gradient boosting where multiple trees have to be calculated for the same training data 
+  // each time with a different target vector. Vectors of idx to allow iteration of intput variables in sorted order are
+  // pre-calculated so that optimal thresholds for splits can be calculated in O(n) for each input variable.
+  // After each split the row idx are partitioned in a left an right part.
+  internal class RegressionTreeBuilder {
+    private readonly IRandom random;
+    private readonly IRegressionProblemData problemData;
+
+    private readonly int nCols;
+    private readonly double[][] x; // all training data (original order from problemData), x is constant
+    private double[] originalY; // the original target labels (from problemData), originalY is constant
+    private double[] curPred; // current predictions for originalY (in case we are using gradient boosting, otherwise = zeros), only necessary for line search
+
+    private double[] y; // training labels (original order from problemData), y can be changed
+
+    private Dictionary<string, double> sumImprovements; // for variable relevance calculation
+
+    private readonly string[] allowedVariables; // all variables in shuffled order
+    private Dictionary<string, int> varName2Index; // maps the variable names to column indexes 
+    private int effectiveVars; // number of variables that are used from allowedVariables
+
+    private int effectiveRows; // number of rows that are used from 
+    private readonly int[][] sortedIdxAll;
+    private readonly int[][] sortedIdx; // random selection from sortedIdxAll (for r < 1.0)
+
+    // helper arrays which are allocated to maximal necessary size only once in the ctor
+    private readonly int[] internalIdx, which, leftTmp, rightTmp;
+    private readonly double[] outx;
+    private readonly int[] outSortedIdx;
+
+    private RegressionTreeModel.TreeNode[] tree; // tree is represented as a flat array of nodes
+    private int curTreeNodeIdx; // the index where the next tree node is stored
+
+    // This class represents information about potential splits.
+    // For each node generated the best splitting variable and threshold as well as
+    // the improvement from the split are stored in a priority queue
+    private class PartitionSplits {
+      public int ParentNodeIdx { get; set; } // the idx of the leaf node representing this partition 
+      public int StartIdx { get; set; } // the start idx of the partition
+      public int EndIndex { get; set; } // the end idx of the partition
+      public string SplittingVariable { get; set; } // the best splitting variable
+      public double SplittingThreshold { get; set; } // the best threshold
+      public double SplittingImprovement { get; set; } // the improvement of the split (for priority queue)
+    }
+
+    // this list hold partitions with the information about the best split (organized as a sorted queue)
+    private readonly IList<PartitionSplits> queue;
+
+    // prepare and allocate buffer variables in ctor
+    public RegressionTreeBuilder(IRegressionProblemData problemData, IRandom random) {
+      this.problemData = problemData;
+      this.random = random;
+
+      var rows = problemData.TrainingIndices.Count();
+
+      this.nCols = problemData.AllowedInputVariables.Count();
+
+      allowedVariables = problemData.AllowedInputVariables.ToArray();
+      varName2Index = new Dictionary<string, int>(allowedVariables.Length);
+      for (int i = 0; i < allowedVariables.Length; i++) varName2Index.Add(allowedVariables[i], i);
+
+      sortedIdxAll = new int[nCols][];
+      sortedIdx = new int[nCols][];
+      sumImprovements = new Dictionary<string, double>();
+      internalIdx = new int[rows];
+      which = new int[rows];
+      leftTmp = new int[rows];
+      rightTmp = new int[rows];
+      outx = new double[rows];
+      outSortedIdx = new int[rows];
+      queue = new List<PartitionSplits>(100);
+
+      x = new double[nCols][];
+      originalY = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices).ToArray();
+      y = new double[originalY.Length];
+      Array.Copy(originalY, y, y.Length); // copy values (originalY is fixed, y is changed in gradient boosting)
+      curPred = Enumerable.Repeat(0.0, y.Length).ToArray(); // zeros
+
+      int col = 0;
+      foreach (var inputVariable in problemData.AllowedInputVariables) {
+        x[col] = problemData.Dataset.GetDoubleValues(inputVariable, problemData.TrainingIndices).ToArray();
+        sortedIdxAll[col] = Enumerable.Range(0, rows).OrderBy(r => x[col][r]).ToArray();
+        sortedIdx[col] = new int[rows];
+        col++;
+      }
+    }
+
+    // specific interface that allows to specify the target labels and the training rows which is necessary when for gradient boosted trees
+    public IRegressionModel CreateRegressionTreeForGradientBoosting(double[] y, double[] curPred, int maxSize, int[] idx, ILossFunction lossFunction, double r = 0.5, double m = 0.5) {
+      Debug.Assert(maxSize > 0);
+      Debug.Assert(r > 0);
+      Debug.Assert(r <= 1.0);
+      Debug.Assert(y.Count() == this.y.Length);
+      Debug.Assert(m > 0);
+      Debug.Assert(m <= 1.0);
+
+      // y and curPred are changed in gradient boosting
+      this.y = y; 
+      this.curPred = curPred; 
+
+      // shuffle row idx
+      HeuristicLab.Random.ListExtensions.ShuffleInPlace(idx, random);
+
+      int nRows = idx.Count();
+
+      // shuffle variable idx
+      HeuristicLab.Random.ListExtensions.ShuffleInPlace(allowedVariables, random);
+
+      // only select a part of the rows and columns randomly
+      effectiveRows = (int)Math.Ceiling(nRows * r);
+      effectiveVars = (int)Math.Ceiling(nCols * m);
+
+      // the which array is used for partitioing row idxs  
+      Array.Clear(which, 0, which.Length);
+
+      // mark selected rows
+      for (int row = 0; row < effectiveRows; row++) {
+        which[idx[row]] = 1; // we use the which vector as a temporary variable here
+        internalIdx[row] = idx[row];
+      }
+
+      for (int col = 0; col < nCols; col++) {
+        int i = 0;
+        for (int row = 0; row < nRows; row++) {
+          if (which[sortedIdxAll[col][row]] > 0) {
+            Debug.Assert(i < effectiveRows);
+            sortedIdx[col][i] = sortedIdxAll[col][row];
+            i++;
+          }
+        }
+      }
+
+      this.tree = new RegressionTreeModel.TreeNode[maxSize];
+      this.queue.Clear();
+      this.curTreeNodeIdx = 0;
+
+      // start out with only one leaf node (constant prediction)
+      // and calculate the best split for this root node and enqueue it into a queue sorted by improvement throught the split
+      // start and end idx are inclusive
+      CreateLeafNode(0, effectiveRows - 1, lossFunction);
+
+      // process the priority queue to complete the tree
+      CreateRegressionTreeFromQueue(maxSize, lossFunction);
+
+      return new RegressionTreeModel(tree.ToArray());
+    }
+
+
+    // processes potential splits from the queue as long as splits are left and the maximum size of the tree is not reached
+    private void CreateRegressionTreeFromQueue(int maxNodes, ILossFunction lossFunction) {
+      while (queue.Any() && curTreeNodeIdx + 1 < maxNodes) { // two nodes are created in each loop
+        var f = queue[queue.Count - 1]; // last element has the largest improvement
+        queue.RemoveAt(queue.Count - 1);
+
+        var startIdx = f.StartIdx;
+        var endIdx = f.EndIndex;
+
+        Debug.Assert(endIdx - startIdx >= 0);
+        Debug.Assert(startIdx >= 0);
+        Debug.Assert(endIdx < internalIdx.Length);
+
+        // split partition into left and right
+        int splitIdx;
+        SplitPartition(f.StartIdx, f.EndIndex, f.SplittingVariable, f.SplittingThreshold, out splitIdx);
+        Debug.Assert(splitIdx + 1 <= endIdx);
+        Debug.Assert(startIdx <= splitIdx);
+
+        // create two leaf nodes (and enqueue best splits for both)
+        var leftTreeIdx = CreateLeafNode(startIdx, splitIdx, lossFunction);
+        var rightTreeIdx = CreateLeafNode(splitIdx + 1, endIdx, lossFunction);
+
+        // overwrite existing leaf node with an internal node
+        tree[f.ParentNodeIdx] = new RegressionTreeModel.TreeNode(f.SplittingVariable, f.SplittingThreshold, leftTreeIdx, rightTreeIdx);
+      }
+    }
+
+
+    // returns the index of the newly created tree node
+    private int CreateLeafNode(int startIdx, int endIdx, ILossFunction lossFunction) {
+      // write a leaf node
+      var val = lossFunction.LineSearch(originalY, curPred, internalIdx, startIdx, endIdx);
+      tree[curTreeNodeIdx] = new RegressionTreeModel.TreeNode(RegressionTreeModel.TreeNode.NO_VARIABLE, val);
+
+      EnqueuePartitionSplit(curTreeNodeIdx, startIdx, endIdx);
+      curTreeNodeIdx++;
+      return curTreeNodeIdx - 1;
+    }
+
+
+    // calculates the optimal split for the partition [startIdx .. endIdx] (inclusive) 
+    // which is represented by the leaf node with the specified nodeIdx
+    private void EnqueuePartitionSplit(int nodeIdx, int startIdx, int endIdx) {
+      double threshold, improvement;
+      string bestVariableName;
+      // only enqueue a new split if there are at least 2 rows left and a split is possible
+      if (startIdx < endIdx &&
+        FindBestVariableAndThreshold(startIdx, endIdx, out threshold, out bestVariableName, out improvement)) {
+        var split = new PartitionSplits() {
+          ParentNodeIdx = nodeIdx,
+          StartIdx = startIdx,
+          EndIndex = endIdx,
+          SplittingThreshold = threshold,
+          SplittingVariable = bestVariableName
+        };
+        InsertSortedQueue(split);
+      }
+    }
+
+
+    // routine for splitting a partition of rows stored in internalIdx between startIdx and endIdx into
+    // a left partition and a right partition using the given splittingVariable and threshold
+    // the splitIdx is the last index of the left partition 
+    // splitIdx + 1 is the first index of the right partition
+    // startIdx and endIdx are inclusive
+    private void SplitPartition(int startIdx, int endIdx, string splittingVar, double threshold, out int splitIdx) {
+      int bestVarIdx = varName2Index[splittingVar];
+      // split - two pass
+
+      // store which index goes into which partition 
+      for (int k = startIdx; k <= endIdx; k++) {
+        if (x[bestVarIdx][internalIdx[k]] <= threshold)
+          which[internalIdx[k]] = -1; // left partition
+        else
+          which[internalIdx[k]] = 1; // right partition
+      }
+
+      // partition sortedIdx for each variable
+      int i;
+      int j;
+      for (int col = 0; col < nCols; col++) {
+        i = 0;
+        j = 0;
+        int k;
+        for (k = startIdx; k <= endIdx; k++) {
+          Debug.Assert(Math.Abs(which[sortedIdx[col][k]]) == 1);
+
+          if (which[sortedIdx[col][k]] < 0) {
+            leftTmp[i++] = sortedIdx[col][k];
+          } else {
+            rightTmp[j++] = sortedIdx[col][k];
+          }
+        }
+        Debug.Assert(i > 0); // at least on element in the left partition
+        Debug.Assert(j > 0); // at least one element in the right partition
+        Debug.Assert(i + j == endIdx - startIdx + 1);
+        k = startIdx;
+        for (int l = 0; l < i; l++) sortedIdx[col][k++] = leftTmp[l];
+        for (int l = 0; l < j; l++) sortedIdx[col][k++] = rightTmp[l];
+      }
+
+      // partition row indices
+      i = startIdx;
+      j = endIdx;
+      while (i <= j) {
+        Debug.Assert(Math.Abs(which[internalIdx[i]]) == 1);
+        Debug.Assert(Math.Abs(which[internalIdx[j]]) == 1);
+        if (which[internalIdx[i]] < 0) i++;
+        else if (which[internalIdx[j]] > 0) j--;
+        else {
+          Debug.Assert(which[internalIdx[i]] > 0);
+          Debug.Assert(which[internalIdx[j]] < 0);
+          // swap
+          int tmp = internalIdx[i];
+          internalIdx[i] = internalIdx[j];
+          internalIdx[j] = tmp;
+          i++;
+          j--;
+        }
+      }
+      Debug.Assert(j + 1 == i);
+      Debug.Assert(i <= endIdx);
+      Debug.Assert(startIdx <= j);
+
+      splitIdx = j;
+    }
+
+    private bool FindBestVariableAndThreshold(int startIdx, int endIdx, out double threshold, out string bestVar, out double improvement) {
+      Debug.Assert(startIdx < endIdx + 1); // at least 2 elements
+
+      int rows = endIdx - startIdx + 1;
+      Debug.Assert(rows >= 2);
+
+      double sumY = 0.0;
+      for (int i = startIdx; i <= endIdx; i++) {
+        sumY += y[internalIdx[i]];
+      }
+
+      // see description of calculation in FindBestThreshold
+      double bestImprovement = 1.0 / rows * sumY * sumY; // any improvement must be larger than this baseline
+      double bestThreshold = double.PositiveInfinity;
+      bestVar = RegressionTreeModel.TreeNode.NO_VARIABLE;
+
+      for (int col = 0; col < effectiveVars; col++) {
+        // sort values for variable to prepare for threshold selection 
+        var curVariable = allowedVariables[col];
+        var curVariableIdx = varName2Index[curVariable];
+        for (int i = startIdx; i <= endIdx; i++) {
+          var sortedI = sortedIdx[curVariableIdx][i];
+          outSortedIdx[i - startIdx] = sortedI;
+          outx[i - startIdx] = x[curVariableIdx][sortedI];
+        }
+
+        double curImprovement;
+        double curThreshold;
+        FindBestThreshold(outx, outSortedIdx, rows, y, sumY, out curThreshold, out curImprovement);
+
+        if (curImprovement > bestImprovement) {
+          bestImprovement = curImprovement;
+          bestThreshold = curThreshold;
+          bestVar = allowedVariables[col];
+        }
+      }
+      if (bestVar == RegressionTreeModel.TreeNode.NO_VARIABLE) {
+        // not successfull
+        threshold = double.PositiveInfinity;
+        improvement = double.NegativeInfinity;
+        return false;
+      } else {
+        UpdateVariableRelevance(bestVar, sumY, bestImprovement, rows);
+        improvement = bestImprovement;
+        threshold = bestThreshold;
+        return true;
+      }
+    }
+
+    // x [0..N-1] contains rows sorted values in the range from [0..rows-1]
+    // sortedIdx [0..N-1] contains the idx of the values in x in the original dataset in the range from [0..rows-1]
+    // rows specifies the number of valid entries in x and sortedIdx
+    // y [0..N-1] contains the target values in original sorting order
+    // sumY is y.Sum()
+    // 
+    // the routine returns the best threshold (x[i] + x[i+1]) / 2 for i = [0 .. rows-2] by calculating the reduction in squared error
+    // additionally the reduction in squared error is returned in bestImprovement
+    // if all elements of x are equal the routing fails to produce a threshold
+    private static void FindBestThreshold(double[] x, int[] sortedIdx, int rows, double[] y, double sumY, out double bestThreshold, out double bestImprovement) {
+      Debug.Assert(rows >= 2);
+
+      double sl = 0.0;
+      double sr = sumY;
+      double nl = 0.0;
+      double nr = rows;
+
+      bestImprovement = 1.0 / rows * sumY * sumY; // this is the baseline for the improvement
+      bestThreshold = double.NegativeInfinity;
+      // for all thresholds
+      // if we have n rows there are n-1 possible splits
+      for (int i = 0; i < rows - 1; i++) {
+        sl += y[sortedIdx[i]];
+        sr -= y[sortedIdx[i]];
+
+        nl++;
+        nr--;
+        Debug.Assert(nl > 0);
+        Debug.Assert(nr > 0);
+
+        if (x[i] < x[i + 1]) { // don't try to split when two elements are equal
+
+          // goal is to find the split with leading to minimal total variance of left and right parts
+          // without partitioning the variance is var(y) = E(y²) - E(y)²  
+          //    = 1/n * sum(y²) - (1/n * sum(y))²
+          //      -------------   ---------------
+          //         constant       baseline for improvement
+          // 
+          // if we split into right and left part the overall variance is the weigthed combination nl/n * var(y_l) + nr/n * var(y_r)  
+          //    = nl/n * (1/nl * sum(y_l²) - (1/nl * sum(y_l))²) + nr/n * (1/nr * sum(y_r²) - (1/nr * sum(y_r))²)
+          //    = 1/n * sum(y_l²) - 1/nl * 1/n * sum(y_l)² + 1/n * sum(y_r²) - 1/nr * 1/n * sum(y_r)²
+          //    = 1/n * (sum(y_l²) + sum(y_r²)) - 1/n * (sum(y_l)² / nl + sum(y_r)² / nr)
+          //    = 1/n * sum(y²) - 1/n * (sum(y_l)² / nl + sum(y_r)² / nr)
+          //      -------------
+          //       not changed by split (and the same for total variance without partitioning)
+          //
+          //   therefore we need to find the maximum value (sum(y_l)² / nl + sum(y_r)² / nr) (ignoring the factor 1/n)
+          //   and this value must be larger than 1/n * sum(y)² to be an improvement over no split 
+
+          double curQuality = sl * sl / nl + sr * sr / nr;
+
+          if (curQuality > bestImprovement) {
+            bestThreshold = (x[i] + x[i + 1]) / 2.0;
+            bestImprovement = curQuality;
+          }
+        }
+      }
+
+      // if all elements where the same then no split can be found
+    }
+
+
+    private void UpdateVariableRelevance(string bestVar, double sumY, double bestImprovement, int rows) {
+      if (string.IsNullOrEmpty(bestVar)) return;
+      // update variable relevance
+      double baseLine = 1.0 / rows * sumY * sumY; // if best improvement is equal to baseline then the split had no effect
+
+      double delta = (bestImprovement - baseLine);
+      double v;
+      if (!sumImprovements.TryGetValue(bestVar, out v)) {
+        sumImprovements[bestVar] = delta;
+      }
+      sumImprovements[bestVar] = v + delta;
+    }
+
+    public IEnumerable<KeyValuePair<string, double>> GetVariableRelevance() {
+      // values are scaled: the most important variable has relevance = 100
+      double scaling = 100 / sumImprovements.Max(t => t.Value);
+      return
+        sumImprovements
+        .Select(t => new KeyValuePair<string, double>(t.Key, t.Value * scaling))
+        .OrderByDescending(t => t.Value);
+    }
+
+
+    // insert a new parition split (find insertion point and start at first element of the queue)
+    // elements are removed from the queue at the last position
+    // O(n), splits could be organized as a heap to improve runtime (see alglib tsort)
+    private void InsertSortedQueue(PartitionSplits split) {
+      // find insertion position
+      int i = 0;
+      while (i < queue.Count && queue[i].SplittingImprovement < split.SplittingImprovement) { i++; }
+
+      queue.Insert(i, split);
+    }
+  }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeModel.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeModel.cs	(revision 13085)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeModel.cs	(revision 13085)
@@ -0,0 +1,155 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Globalization;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item("RegressionTreeModel", "Represents a decision tree for regression.")]
+  public sealed class RegressionTreeModel : NamedItem, IRegressionModel {
+
+    // trees are represented as a flat array    
+    internal struct TreeNode {
+      public readonly static string NO_VARIABLE = null;
+
+      public TreeNode(string varName, double val, int leftIdx = -1, int rightIdx = -1)
+        : this() {
+        VarName = varName;
+        Val = val;
+        LeftIdx = leftIdx;
+        RightIdx = rightIdx;
+      }
+
+      public string VarName { get; private set; } // name of the variable for splitting or NO_VARIABLE if terminal node
+      public double Val { get; private set; } // threshold
+      public int LeftIdx { get; private set; }
+      public int RightIdx { get; private set; }
+
+      // necessary because the default implementation of GetHashCode for structs in .NET would only return the hashcode of val here
+      public override int GetHashCode() {
+        return LeftIdx ^ RightIdx ^ Val.GetHashCode();
+      }
+      // necessary because of GetHashCode override
+      public override bool Equals(object obj) {
+        if (obj is TreeNode) {
+          var other = (TreeNode)obj;
+          return Val.Equals(other.Val) &&
+            LeftIdx.Equals(other.LeftIdx) &&
+            RightIdx.Equals(other.RightIdx) &&
+            EqualStrings(VarName, other.VarName);
+        } else {
+          return false;
+        }
+      }
+
+      private bool EqualStrings(string a, string b) {
+        return (a == null && b == null) ||
+               (a != null && b != null && a.Equals(b));
+      }
+    }
+
+    // not storable!
+    private TreeNode[] tree;
+
+    [Storable]
+    // to prevent storing the references to data caches in nodes
+    // TODO seemingly it is bad (performance-wise) to persist tuples (tuples are used as keys in a dictionary)
+    private Tuple<string, double, int, int>[] SerializedTree {
+      get { return tree.Select(t => Tuple.Create(t.VarName, t.Val, t.LeftIdx, t.RightIdx)).ToArray(); }
+      set { this.tree = value.Select(t => new TreeNode(t.Item1, t.Item2, t.Item3, t.Item4)).ToArray(); }
+    }
+
+    [StorableConstructor]
+    private RegressionTreeModel(bool serializing) : base(serializing) { }
+    // cloning ctor
+    private RegressionTreeModel(RegressionTreeModel original, Cloner cloner)
+      : base(original, cloner) {
+      if (original.tree != null) {
+        this.tree = new TreeNode[original.tree.Length];
+        Array.Copy(original.tree, this.tree, this.tree.Length);
+      }
+    }
+
+    internal RegressionTreeModel(TreeNode[] tree)
+      : base("RegressionTreeModel", "Represents a decision tree for regression.") {
+      this.tree = tree;
+    }
+
+    private static double GetPredictionForRow(TreeNode[] t, ReadOnlyCollection<double>[] columnCache, int nodeIdx, int row) {
+      while (nodeIdx != -1) {
+        var node = t[nodeIdx];
+        if (node.VarName == TreeNode.NO_VARIABLE)
+          return node.Val;
+
+        if (columnCache[nodeIdx][row] <= node.Val)
+          nodeIdx = node.LeftIdx;
+        else
+          nodeIdx = node.RightIdx;
+      }
+      throw new InvalidOperationException("Invalid tree in RegressionTreeModel");
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RegressionTreeModel(this, cloner);
+    }
+
+    public IEnumerable<double> GetEstimatedValues(IDataset ds, IEnumerable<int> rows) {
+      // lookup columns for variableNames in one pass over the tree to speed up evaluation later on
+      ReadOnlyCollection<double>[] columnCache = new ReadOnlyCollection<double>[tree.Length];
+
+      for (int i = 0; i < tree.Length; i++) {
+        if (tree[i].VarName != TreeNode.NO_VARIABLE) {
+          columnCache[i] = ds.GetReadOnlyDoubleValues(tree[i].VarName);
+        }
+      }
+      return rows.Select(r => GetPredictionForRow(tree, columnCache, 0, r));
+    }
+
+    public IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RegressionSolution(this, new RegressionProblemData(problemData));
+    }
+
+    // mainly for debugging
+    public override string ToString() {
+      return TreeToString(0, "");
+    }
+
+    private string TreeToString(int idx, string part) {
+      var n = tree[idx];
+      if (n.VarName == TreeNode.NO_VARIABLE) {
+        return string.Format(CultureInfo.InvariantCulture, "{0} -> {1:F}{2}", part, n.Val, Environment.NewLine);
+      } else {
+        return
+          TreeToString(n.LeftIdx, string.Format(CultureInfo.InvariantCulture, "{0}{1}{2} <= {3:F}", part, string.IsNullOrEmpty(part) ? "" : " and ", n.VarName, n.Val))
+        + TreeToString(n.RightIdx, string.Format(CultureInfo.InvariantCulture, "{0}{1}{2} >  {3:F}", part, string.IsNullOrEmpty(part) ? "" : " and ", n.VarName, n.Val));
+      }
+    }
+  }
+}
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj	(revision 13085)
@@ -269,4 +269,5 @@
       <SubType>Code</SubType>
     </Compile>
+    <Compile Include="GaussianProcess\GaussianProcessCovarianceOptimizationProblem.cs" />
     <Compile Include="GaussianProcess\Util.cs" />
     <Compile Include="GaussianProcess\IMeanFunction.cs" />
@@ -275,4 +276,16 @@
     <Compile Include="GaussianProcess\GaussianProcessRegressionSolution.cs" />
     <Compile Include="GaussianProcess\ICovarianceFunction.cs" />
+    <Compile Include="GradientBoostedTrees\GradientBoostedTreesModelSurrogate.cs" />
+    <Compile Include="GradientBoostedTrees\GradientBoostedTreesAlgorithm.cs" />
+    <Compile Include="GradientBoostedTrees\GradientBoostedTreesAlgorithmStatic.cs" />
+    <Compile Include="GradientBoostedTrees\GradientBoostedTreesModel.cs" />
+    <Compile Include="GradientBoostedTrees\LossFunctions\AbsoluteErrorLoss.cs" />
+    <Compile Include="GradientBoostedTrees\LossFunctions\QuantileRegressionLoss.cs" />
+    <Compile Include="GradientBoostedTrees\LossFunctions\ILossFunction.cs" />
+    <Compile Include="GradientBoostedTrees\LossFunctions\LogisticRegressionLoss.cs" />
+    <Compile Include="GradientBoostedTrees\LossFunctions\RelativeErrorLoss.cs" />
+    <Compile Include="GradientBoostedTrees\LossFunctions\SquaredErrorLoss.cs" />
+    <Compile Include="GradientBoostedTrees\RegressionTreeBuilder.cs" />
+    <Compile Include="GradientBoostedTrees\RegressionTreeModel.cs" />
     <Compile Include="Interfaces\IGaussianProcessClassificationModelCreator.cs" />
     <Compile Include="Interfaces\IGaussianProcessRegressionModelCreator.cs" />
@@ -353,4 +366,5 @@
     <Compile Include="RandomForest\RandomForestRegression.cs" />
     <Compile Include="RandomForest\RandomForestRegressionSolution.cs" />
+    <Compile Include="RandomForest\RandomForestUtil.cs" />
     <Compile Include="SupportVectorMachine\SupportVectorClassification.cs" />
     <Compile Include="SupportVectorMachine\SupportVectorClassificationSolution.cs" />
@@ -366,4 +380,110 @@
     </Compile>
     <Compile Include="TimeSeries\AutoregressiveModeling.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\HeuristicLab.Algorithms.GradientDescent\3.3\HeuristicLab.Algorithms.GradientDescent-3.3.csproj">
+      <Project>{1256B945-EEA9-4BE4-9880-76B5B113F089}</Project>
+      <Name>HeuristicLab.Algorithms.GradientDescent-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Analysis\3.3\HeuristicLab.Analysis-3.3.csproj">
+      <Project>{887425B4-4348-49ED-A457-B7D2C26DDBF9}</Project>
+      <Name>HeuristicLab.Analysis-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Collections\3.3\HeuristicLab.Collections-3.3.csproj">
+      <Project>{958B43BC-CC5C-4FA2-8628-2B3B01D890B6}</Project>
+      <Name>HeuristicLab.Collections-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Common.Resources\3.3\HeuristicLab.Common.Resources-3.3.csproj">
+      <Project>{0E27A536-1C4A-4624-A65E-DC4F4F23E3E1}</Project>
+      <Name>HeuristicLab.Common.Resources-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Common\3.3\HeuristicLab.Common-3.3.csproj">
+      <Project>{A9AD58B9-3EF9-4CC1-97E5-8D909039FF5C}</Project>
+      <Name>HeuristicLab.Common-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Core\3.3\HeuristicLab.Core-3.3.csproj">
+      <Project>{C36BD924-A541-4A00-AFA8-41701378DDC5}</Project>
+      <Name>HeuristicLab.Core-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Data\3.3\HeuristicLab.Data-3.3.csproj">
+      <Project>{BBAB9DF5-5EF3-4BA8-ADE9-B36E82114937}</Project>
+      <Name>HeuristicLab.Data-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Encodings.RealVectorEncoding\3.3\HeuristicLab.Encodings.RealVectorEncoding-3.3.csproj">
+      <Project>{BB6D334A-4BB6-4674-9883-31A6EBB32CAB}</Project>
+      <Name>HeuristicLab.Encodings.RealVectorEncoding-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Encodings.SymbolicExpressionTreeEncoding\3.4\HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4.csproj">
+      <Project>{06D4A186-9319-48A0-BADE-A2058D462EEA}</Project>
+      <Name>HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Operators\3.3\HeuristicLab.Operators-3.3.csproj">
+      <Project>{23DA7FF4-D5B8-41B6-AA96-F0561D24F3EE}</Project>
+      <Name>HeuristicLab.Operators-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Optimization\3.3\HeuristicLab.Optimization-3.3.csproj">
+      <Project>{14AB8D24-25BC-400C-A846-4627AA945192}</Project>
+      <Name>HeuristicLab.Optimization-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Parameters\3.3\HeuristicLab.Parameters-3.3.csproj">
+      <Project>{56F9106A-079F-4C61-92F6-86A84C2D84B7}</Project>
+      <Name>HeuristicLab.Parameters-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Persistence\3.3\HeuristicLab.Persistence-3.3.csproj">
+      <Project>{102BC7D3-0EF9-439C-8F6D-96FF0FDB8E1B}</Project>
+      <Name>HeuristicLab.Persistence-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.PluginInfrastructure\3.3\HeuristicLab.PluginInfrastructure-3.3.csproj">
+      <Project>{94186A6A-5176-4402-AE83-886557B53CCA}</Project>
+      <Name>HeuristicLab.PluginInfrastructure-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.DataAnalysis.Symbolic.Classification\3.4\HeuristicLab.Problems.DataAnalysis.Symbolic.Classification-3.4.csproj">
+      <Project>{05BAE4E1-A9FA-4644-AA77-42558720159E}</Project>
+      <Name>HeuristicLab.Problems.DataAnalysis.Symbolic.Classification-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.DataAnalysis.Symbolic.Regression\3.4\HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4.csproj">
+      <Project>{5AC82412-911B-4FA2-A013-EDC5E3F3FCC2}</Project>
+      <Name>HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.DataAnalysis.Symbolic.TimeSeriesPrognosis\3.4\HeuristicLab.Problems.DataAnalysis.Symbolic.TimeSeriesPrognosis-3.4.csproj">
+      <Project>{07486E68-1517-4B9D-A58D-A38E99AE71AB}</Project>
+      <Name>HeuristicLab.Problems.DataAnalysis.Symbolic.TimeSeriesPrognosis-3.4</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.DataAnalysis.Symbolic\3.4\HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj">
+      <Project>{3D28463F-EC96-4D82-AFEE-38BE91A0CA00}</Project>
+      <Name>HeuristicLab.Problems.DataAnalysis.Symbolic-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.DataAnalysis\3.4\HeuristicLab.Problems.DataAnalysis-3.4.csproj">
+      <Project>{DF87C13E-A889-46FF-8153-66DCAA8C5674}</Project>
+      <Name>HeuristicLab.Problems.DataAnalysis-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.Instances\3.3\HeuristicLab.Problems.Instances-3.3.csproj">
+      <Project>{3540E29E-4793-49E7-8EE2-FEA7F61C3994}</Project>
+      <Name>HeuristicLab.Problems.Instances-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Random\3.3\HeuristicLab.Random-3.3.csproj">
+      <Project>{F4539FB6-4708-40C9-BE64-0A1390AEA197}</Project>
+      <Name>HeuristicLab.Random-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
   </ItemGroup>
   <ItemGroup>
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IDataAnalysisAlgorithm.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IDataAnalysisAlgorithm.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IDataAnalysisAlgorithm.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessClassificationModelCreator.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessClassificationModelCreator.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessClassificationModelCreator.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModel.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModel.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModel.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -34,5 +34,5 @@
     double[] HyperparameterGradients { get; }
 
-    IEnumerable<double> GetEstimatedVariance(Dataset ds, IEnumerable<int> rows);
+    IEnumerable<double> GetEstimatedVariance(IDataset ds, IEnumerable<int> rows);
     void FixParameters();
   }
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModelCreator.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModelCreator.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModelCreator.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessRegressionModelCreator.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessRegressionModelCreator.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessRegressionModelCreator.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaClassificationSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaClassificationSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaClassificationSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaModel.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaModel.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaModel.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -27,5 +27,5 @@
     new INcaClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData);
 
-    double[,] Reduce(Dataset dataset, IEnumerable<int> rows);
+    double[,] Reduce(IDataset dataset, IEnumerable<int> rows);
   }
 }
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourClassificationSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourClassificationSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourClassificationSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourModel.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourModel.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourModel.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourRegressionSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourRegressionSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourRegressionSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkClassificationSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkClassificationSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkClassificationSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleClassificationSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleClassificationSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleClassificationSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleModel.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleModel.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleModel.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleRegressionSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleRegressionSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleRegressionSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkModel.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkModel.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkModel.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkRegressionSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkRegressionSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkRegressionSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestClassificationSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestClassificationSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestClassificationSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestModel.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestModel.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestModel.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestRegressionSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestRegressionSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestRegressionSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineModel.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineModel.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineModel.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/AlglibUtil.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/AlglibUtil.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/AlglibUtil.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -26,5 +26,5 @@
 namespace HeuristicLab.Algorithms.DataAnalysis {
   public static class AlglibUtil {
-    public static double[,] PrepareInputMatrix(Dataset dataset, IEnumerable<string> variables, IEnumerable<int> rows) {
+    public static double[,] PrepareInputMatrix(IDataset dataset, IEnumerable<string> variables, IEnumerable<int> rows) {
       List<string> variablesList = variables.ToList();
       List<int> rowsList = rows.ToList();
@@ -45,5 +45,5 @@
       return matrix;
     }
-    public static double[,] PrepareAndScaleInputMatrix(Dataset dataset, IEnumerable<string> variables, IEnumerable<int> rows, Scaling scaling) {
+    public static double[,] PrepareAndScaleInputMatrix(IDataset dataset, IEnumerable<string> variables, IEnumerable<int> rows, Scaling scaling) {
       List<string> variablesList = variables.ToList();
       List<int> rowsList = rows.ToList();
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearDiscriminantAnalysis.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearDiscriminantAnalysis.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearDiscriminantAnalysis.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -37,5 +37,5 @@
   /// </summary>
   [Item("Linear Discriminant Analysis", "Linear discriminant analysis classification algorithm (wrapper for ALGLIB).")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 100)]
   [StorableClass]
   public sealed class LinearDiscriminantAnalysis : FixedDataAnalysisAlgorithm<IClassificationProblem> {
@@ -65,5 +65,5 @@
 
     public static IClassificationSolution CreateLinearDiscriminantAnalysisSolution(IClassificationProblemData problemData) {
-      Dataset dataset = problemData.Dataset;
+      var dataset = problemData.Dataset;
       string targetVariable = problemData.TargetVariable;
       IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegression.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegression.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegression.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -38,5 +38,5 @@
   /// </summary>
   [Item("Linear Regression", "Linear regression data analysis algorithm (wrapper for ALGLIB).")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 100)]
   [StorableClass]
   public sealed class LinearRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
@@ -69,5 +69,5 @@
 
     public static ISymbolicRegressionSolution CreateLinearRegressionSolution(IRegressionProblemData problemData, out double rmsError, out double cvRmsError) {
-      Dataset dataset = problemData.Dataset;
+      var dataset = problemData.Dataset;
       string targetVariable = problemData.TargetVariable;
       IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassification.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassification.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassification.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -35,5 +35,5 @@
   /// </summary>
   [Item("Multinomial Logit Classification", "Multinomial logit classification data analysis algorithm (wrapper for ALGLIB).")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 180)]
   [StorableClass]
   public sealed class MultiNomialLogitClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
@@ -66,5 +66,5 @@
 
     public static IClassificationSolution CreateLogitClassificationSolution(IClassificationProblemData problemData, out double rmsError, out double relClassError) {
-      Dataset dataset = problemData.Dataset;
+      var dataset = problemData.Dataset;
       string targetVariable = problemData.TargetVariable;
       IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassificationSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassificationSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassificationSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitModel.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitModel.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitModel.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -82,5 +82,5 @@
     }
 
-    public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
+    public IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
       double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
 
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/Scaling.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/Scaling.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/Scaling.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -41,5 +41,5 @@
         scalingParameters.Add(pair.Key, Tuple.Create(pair.Value.Item1, pair.Value.Item2));
     }
-    public Scaling(Dataset ds, IEnumerable<string> variables, IEnumerable<int> rows) {
+    public Scaling(IDataset ds, IEnumerable<string> variables, IEnumerable<int> rows) {
       foreach (var variable in variables) {
         var values = ds.GetDoubleValues(variable, rows);
@@ -54,5 +54,5 @@
     }
 
-    public IEnumerable<double> GetScaledValues(Dataset ds, string variable, IEnumerable<int> rows) {
+    public IEnumerable<double> GetScaledValues(IDataset ds, string variable, IEnumerable<int> rows) {
       double min = scalingParameters[variable].Item1;
       double max = scalingParameters[variable].Item2;
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/INcaInitializer.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/INcaInitializer.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/INcaInitializer.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/LdaInitializer.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/LdaInitializer.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/LdaInitializer.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/NcaInitializer.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/NcaInitializer.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/NcaInitializer.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/PcaInitializer.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/PcaInitializer.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/PcaInitializer.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/RandomInitializer.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/RandomInitializer.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/RandomInitializer.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Matrix.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Matrix.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Matrix.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/ModelCreation/INcaModelCreator.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/ModelCreation/INcaModelCreator.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/ModelCreation/INcaModelCreator.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/ModelCreation/NcaModelCreator.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/ModelCreation/NcaModelCreator.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/ModelCreation/NcaModelCreator.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaAlgorithm.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaAlgorithm.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaAlgorithm.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -43,5 +43,5 @@
 with additional regularizations described in Z. Yang, J. Laaksonen. 2007.
 Regularized Neighborhood Component Analysis. Lecture Notes in Computer Science, 4522. pp. 253-262.")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 170)]
   [StorableClass]
   public sealed class NcaAlgorithm : EngineAlgorithm {
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaClassificationSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaClassificationSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaClassificationSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaGradientCalculator.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaGradientCalculator.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaGradientCalculator.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -28,4 +28,5 @@
 using HeuristicLab.Encodings.RealVectorEncoding;
 using HeuristicLab.Operators;
+using HeuristicLab.Optimization;
 using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
@@ -35,5 +36,5 @@
   [Item("NcaGradientCalculator", "Calculates the quality and gradient of a certain NCA matrix.")]
   [StorableClass]
-  public class NcaGradientCalculator : SingleSuccessorOperator {
+  public class NcaGradientCalculator : SingleSuccessorOperator, ISingleObjectiveOperator {
 
     #region Parameter Properties
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -56,5 +56,5 @@
       this.classValues = (double[])original.classValues.Clone();
     }
-    public NcaModel(int k, double[,] transformationMatrix, Dataset dataset, IEnumerable<int> rows, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues) {
+    public NcaModel(int k, double[,] transformationMatrix, IDataset dataset, IEnumerable<int> rows, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues) {
       Name = ItemName;
       Description = ItemDescription;
@@ -72,5 +72,5 @@
     }
 
-    public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
+    public IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
       var ds = ReduceDataset(dataset, rows);
       return nnModel.GetEstimatedClassValues(ds, Enumerable.Range(0, ds.Rows));
@@ -85,5 +85,5 @@
     }
 
-    public double[,] Reduce(Dataset dataset, IEnumerable<int> rows) {
+    public double[,] Reduce(IDataset dataset, IEnumerable<int> rows) {
       var data = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
 
@@ -100,5 +100,5 @@
     }
 
-    public Dataset ReduceDataset(Dataset dataset, IEnumerable<int> rows) {
+    public Dataset ReduceDataset(IDataset dataset, IEnumerable<int> rows) {
       return new Dataset(Enumerable
           .Range(0, transformationMatrix.GetLength(1))
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/SolutionCreation/INcaSolutionCreator.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/SolutionCreation/INcaSolutionCreator.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/SolutionCreation/INcaSolutionCreator.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/SolutionCreation/NcaSolutionCreator.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/SolutionCreation/NcaSolutionCreator.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/SolutionCreation/NcaSolutionCreator.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -35,5 +35,5 @@
   /// </summary>
   [Item("Nearest Neighbour Classification", "Nearest neighbour classification data analysis algorithm (wrapper for ALGLIB).")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 150)]
   [StorableClass]
   public sealed class NearestNeighbourClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassificationSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassificationSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassificationSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -96,5 +96,5 @@
         this.classValues = (double[])original.classValues.Clone();
     }
-    public NearestNeighbourModel(Dataset dataset, IEnumerable<int> rows, int k, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues = null) {
+    public NearestNeighbourModel(IDataset dataset, IEnumerable<int> rows, int k, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues = null) {
       Name = ItemName;
       Description = ItemDescription;
@@ -135,5 +135,5 @@
     }
 
-    public IEnumerable<double> GetEstimatedValues(Dataset dataset, IEnumerable<int> rows) {
+    public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
       double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
 
@@ -163,5 +163,5 @@
     }
 
-    public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
+    public IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
       if (classValues == null) throw new InvalidOperationException("No class values are defined.");
       double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -34,5 +34,5 @@
   /// </summary>
   [Item("Nearest Neighbour Regression", "Nearest neighbour regression data analysis algorithm (wrapper for ALGLIB).")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 150)]
   [StorableClass]
   public sealed class NearestNeighbourRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegressionSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegressionSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegressionSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -36,5 +36,5 @@
   /// </summary>
   [Item("Neural Network Classification", "Neural network classification data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/neuralnetworks.php")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 130)]
   [StorableClass]
   public sealed class NeuralNetworkClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
@@ -179,5 +179,5 @@
     public static IClassificationSolution CreateNeuralNetworkClassificationSolution(IClassificationProblemData problemData, int nLayers, int nHiddenNodes1, int nHiddenNodes2, double decay, int restarts,
       out double rmsError, out double avgRelError, out double relClassError) {
-      Dataset dataset = problemData.Dataset;
+      var dataset = problemData.Dataset;
       string targetVariable = problemData.TargetVariable;
       IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassificationSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassificationSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassificationSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassification.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassification.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassification.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -36,5 +36,5 @@
   /// </summary>
   [Item("Neural Network Ensemble Classification", "Neural network ensemble classification data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/mlpensembles.php")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 140)]
   [StorableClass]
   public sealed class NeuralNetworkEnsembleClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
@@ -124,5 +124,5 @@
     public NeuralNetworkEnsembleClassification()
       : base() {
-        var validHiddenLayerValues = new ItemSet<IntValue>(new IntValue[] { 
+      var validHiddenLayerValues = new ItemSet<IntValue>(new IntValue[] { 
         (IntValue)new IntValue(0).AsReadOnly(), 
         (IntValue)new IntValue(1).AsReadOnly(), 
@@ -165,5 +165,5 @@
     public static IClassificationSolution CreateNeuralNetworkEnsembleClassificationSolution(IClassificationProblemData problemData, int ensembleSize, int nLayers, int nHiddenNodes1, int nHiddenNodes2, double decay, int restarts,
       out double rmsError, out double avgRelError, out double relClassError) {
-      Dataset dataset = problemData.Dataset;
+      var dataset = problemData.Dataset;
       string targetVariable = problemData.TargetVariable;
       IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassificationSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassificationSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassificationSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleModel.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleModel.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleModel.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -86,5 +86,5 @@
     }
 
-    public IEnumerable<double> GetEstimatedValues(Dataset dataset, IEnumerable<int> rows) {
+    public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
       double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
 
@@ -103,5 +103,5 @@
     }
 
-    public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
+    public IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
       double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
 
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegression.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegression.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegression.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -36,5 +36,5 @@
   /// </summary>
   [Item("Neural Network Ensemble Regression", "Neural network ensemble regression data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/mlpensembles.php")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 140)]
   [StorableClass]
   public sealed class NeuralNetworkEnsembleRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
@@ -124,5 +124,5 @@
     public NeuralNetworkEnsembleRegression()
       : base() {
-        var validHiddenLayerValues = new ItemSet<IntValue>(new IntValue[] { 
+      var validHiddenLayerValues = new ItemSet<IntValue>(new IntValue[] { 
         (IntValue)new IntValue(0).AsReadOnly(), 
         (IntValue)new IntValue(1).AsReadOnly(), 
@@ -164,5 +164,5 @@
     public static IRegressionSolution CreateNeuralNetworkEnsembleRegressionSolution(IRegressionProblemData problemData, int ensembleSize, int nLayers, int nHiddenNodes1, int nHiddenNodes2, double decay, int restarts,
       out double rmsError, out double avgRelError) {
-      Dataset dataset = problemData.Dataset;
+      var dataset = problemData.Dataset;
       string targetVariable = problemData.TargetVariable;
       IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegressionSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegressionSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegressionSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkModel.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkModel.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkModel.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -94,5 +94,5 @@
     }
 
-    public IEnumerable<double> GetEstimatedValues(Dataset dataset, IEnumerable<int> rows) {
+    public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
       double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
 
@@ -111,5 +111,5 @@
     }
 
-    public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
+    public IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
       double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
 
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -36,5 +36,5 @@
   /// </summary>
   [Item("Neural Network Regression", "Neural network regression data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/neuralnetworks.php")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 130)]
   [StorableClass]
   public sealed class NeuralNetworkRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
@@ -180,5 +180,5 @@
     public static IRegressionSolution CreateNeuralNetworkRegressionSolution(IRegressionProblemData problemData, int nLayers, int nHiddenNodes1, int nHiddenNodes2, double decay, int restarts,
       out double rmsError, out double avgRelError) {
-      Dataset dataset = problemData.Dataset;
+      var dataset = problemData.Dataset;
       string targetVariable = problemData.TargetVariable;
       IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegressionSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegressionSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegressionSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -26,5 +26,5 @@
   /// Plugin class for HeuristicLab.Algorithms.DataAnalysis plugin.
   /// </summary>
-  [Plugin("HeuristicLab.Algorithms.DataAnalysis", "Provides wrappers for data analysis algorithms implemented in external libraries (linear regression, linear discriminant analysis, k-means clustering, support vector classification and regression)", "3.4.5.$WCREV$")]
+  [Plugin("HeuristicLab.Algorithms.DataAnalysis", "Provides wrappers for data analysis algorithms implemented in external libraries (linear regression, linear discriminant analysis, k-means clustering, support vector classification and regression)", "3.4.8.$WCREV$")]
   [PluginFile("HeuristicLab.Algorithms.DataAnalysis-3.4.dll", PluginFileType.Assembly)]
   [PluginDependency("HeuristicLab.ALGLIB", "3.7.0")]
@@ -47,4 +47,5 @@
   [PluginDependency("HeuristicLab.Problems.DataAnalysis.Symbolic.Regression", "3.4")]
   [PluginDependency("HeuristicLab.Problems.DataAnalysis.Symbolic.TimeSeriesPrognosis", "3.4")]
+  [PluginDependency("HeuristicLab.Problems.Instances", "3.3")]
   [PluginDependency("HeuristicLab.LibSVM", "3.12")]
   [PluginDependency("HeuristicLab.Random", "3.3")]
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Properties/AssemblyInfo.cs.frame
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Properties/AssemblyInfo.cs.frame	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Properties/AssemblyInfo.cs.frame	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -31,5 +31,5 @@
 [assembly: AssemblyCompany("")]
 [assembly: AssemblyProduct("HeuristicLab")]
-[assembly: AssemblyCopyright("(c) 2002-2013 HEAL")]
+[assembly: AssemblyCopyright("(c) 2002-2015 HEAL")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
@@ -53,3 +53,3 @@
 // by using the '*' as shown below:
 [assembly: AssemblyVersion("3.4.0.0")]
-[assembly: AssemblyFileVersion("3.4.5.$WCREV$")]
+[assembly: AssemblyFileVersion("3.4.8.$WCREV$")]
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassification.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassification.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassification.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -20,7 +20,4 @@
 #endregion
 
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -36,5 +33,5 @@
   /// </summary>
   [Item("Random Forest Classification", "Random forest classification data analysis algorithm (wrapper for ALGLIB).")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 120)]
   [StorableClass]
   public sealed class RandomForestClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
@@ -131,45 +128,6 @@
     public static IClassificationSolution CreateRandomForestClassificationSolution(IClassificationProblemData problemData, int nTrees, double r, double m, int seed,
       out double rmsError, out double relClassificationError, out double outOfBagRmsError, out double outOfBagRelClassificationError) {
-      if (r <= 0 || r > 1) throw new ArgumentException("The R parameter in the random forest regression must be between 0 and 1.");
-      if (m <= 0 || m > 1) throw new ArgumentException("The M parameter in the random forest regression must be between 0 and 1.");
-
-      alglib.math.rndobject = new System.Random(seed);
-
-      Dataset dataset = problemData.Dataset;
-      string targetVariable = problemData.TargetVariable;
-      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
-      IEnumerable<int> rows = problemData.TrainingIndices;
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
-      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
-        throw new NotSupportedException("Random forest classification does not support NaN or infinity values in the input dataset.");
-
-      int info = 0;
-      alglib.decisionforest dForest = new alglib.decisionforest();
-      alglib.dfreport rep = new alglib.dfreport(); ;
-      int nRows = inputMatrix.GetLength(0);
-      int nColumns = inputMatrix.GetLength(1);
-      int sampleSize = Math.Max((int)Math.Round(r * nRows), 1);
-      int nFeatures = Math.Max((int)Math.Round(m * (nColumns - 1)), 1);
-
-
-      double[] classValues = problemData.ClassValues.ToArray();
-      int nClasses = problemData.Classes;
-      // map original class values to values [0..nClasses-1]
-      Dictionary<double, double> classIndices = new Dictionary<double, double>();
-      for (int i = 0; i < nClasses; i++) {
-        classIndices[classValues[i]] = i;
-      }
-      for (int row = 0; row < nRows; row++) {
-        inputMatrix[row, nColumns - 1] = classIndices[inputMatrix[row, nColumns - 1]];
-      }
-      // execute random forest algorithm      
-      alglib.dforest.dfbuildinternal(inputMatrix, nRows, nColumns - 1, nClasses, nTrees, sampleSize, nFeatures, alglib.dforest.dfusestrongsplits + alglib.dforest.dfuseevs, ref info, dForest.innerobj, rep.innerobj);
-      if (info != 1) throw new ArgumentException("Error in calculation of random forest classification solution");
-
-      rmsError = rep.rmserror;
-      outOfBagRmsError = rep.oobrmserror;
-      relClassificationError = rep.relclserror;
-      outOfBagRelClassificationError = rep.oobrelclserror;
-      return new RandomForestClassificationSolution((IClassificationProblemData)problemData.Clone(), new RandomForestModel(dForest, targetVariable, allowedInputVariables, classValues));
+      var model = RandomForestModel.CreateClassificationModel(problemData, nTrees, r, m, seed, out rmsError, out relClassificationError, out outOfBagRmsError, out outOfBagRelClassificationError);
+      return new RandomForestClassificationSolution((IClassificationProblemData)problemData.Clone(), model);
     }
     #endregion
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassificationSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassificationSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassificationSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -35,28 +35,35 @@
   [Item("RandomForestModel", "Represents a random forest for regression and classification.")]
   public sealed class RandomForestModel : NamedItem, IRandomForestModel {
-
+    // not persisted
     private alglib.decisionforest randomForest;
-    public alglib.decisionforest RandomForest {
-      get { return randomForest; }
-      set {
-        if (value != randomForest) {
-          if (value == null) throw new ArgumentNullException();
-          randomForest = value;
-          OnChanged(EventArgs.Empty);
-        }
-      }
-    }
-
-    [Storable]
-    private string targetVariable;
-    [Storable]
-    private string[] allowedInputVariables;
+    private alglib.decisionforest RandomForest {
+      get {
+        // recalculate lazily
+        if (randomForest.innerobj.trees == null || randomForest.innerobj.trees.Length == 0) RecalculateModel();
+        return randomForest;
+      }
+    }
+
+    // instead of storing the data of the model itself
+    // we instead only store data necessary to recalculate the same model lazily on demand
+    [Storable]
+    private int seed;
+    [Storable]
+    private IDataAnalysisProblemData originalTrainingData;
     [Storable]
     private double[] classValues;
+    [Storable]
+    private int nTrees;
+    [Storable]
+    private double r;
+    [Storable]
+    private double m;
+
+
     [StorableConstructor]
     private RandomForestModel(bool deserializing)
       : base(deserializing) {
-      if (deserializing)
-        randomForest = new alglib.decisionforest();
+      // for backwards compatibility (loading old solutions)
+      randomForest = new alglib.decisionforest();
     }
     private RandomForestModel(RandomForestModel original, Cloner cloner)
@@ -67,19 +74,36 @@
       randomForest.innerobj.ntrees = original.randomForest.innerobj.ntrees;
       randomForest.innerobj.nvars = original.randomForest.innerobj.nvars;
-      randomForest.innerobj.trees = (double[])original.randomForest.innerobj.trees.Clone();
-      targetVariable = original.targetVariable;
-      allowedInputVariables = (string[])original.allowedInputVariables.Clone();
-      if (original.classValues != null)
-        this.classValues = (double[])original.classValues.Clone();
-    }
-    public RandomForestModel(alglib.decisionforest randomForest, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues = null)
+      // we assume that the trees array (double[]) is immutable in alglib
+      randomForest.innerobj.trees = original.randomForest.innerobj.trees;
+
+      // allowedInputVariables is immutable so we don't need to clone
+      allowedInputVariables = original.allowedInputVariables;
+
+      // clone data which is necessary to rebuild the model
+      this.seed = original.seed;
+      this.originalTrainingData = cloner.Clone(original.originalTrainingData);
+      // classvalues is immutable so we don't need to clone
+      this.classValues = original.classValues;
+      this.nTrees = original.nTrees;
+      this.r = original.r;
+      this.m = original.m;
+    }
+
+    // random forest models can only be created through the static factory methods CreateRegressionModel and CreateClassificationModel
+    private RandomForestModel(alglib.decisionforest randomForest,
+      int seed, IDataAnalysisProblemData originalTrainingData,
+      int nTrees, double r, double m, double[] classValues = null)
       : base() {
       this.name = ItemName;
       this.description = ItemDescription;
+      // the model itself
       this.randomForest = randomForest;
-      this.targetVariable = targetVariable;
-      this.allowedInputVariables = allowedInputVariables.ToArray();
-      if (classValues != null)
-        this.classValues = (double[])classValues.Clone();
+      // data which is necessary for recalculation of the model
+      this.seed = seed;
+      this.originalTrainingData = (IDataAnalysisProblemData)originalTrainingData.Clone();
+      this.classValues = classValues;
+      this.nTrees = nTrees;
+      this.r = r;
+      this.m = m;
     }
 
@@ -88,6 +112,24 @@
     }
 
-    public IEnumerable<double> GetEstimatedValues(Dataset dataset, IEnumerable<int> rows) {
-      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+    private void RecalculateModel() {
+      double rmsError, oobRmsError, relClassError, oobRelClassError;
+      var regressionProblemData = originalTrainingData as IRegressionProblemData;
+      var classificationProblemData = originalTrainingData as IClassificationProblemData;
+      if (regressionProblemData != null) {
+        var model = CreateRegressionModel(regressionProblemData,
+                                              nTrees, r, m, seed, out rmsError, out oobRmsError,
+                                              out relClassError, out oobRelClassError);
+        randomForest = model.randomForest;
+      } else if (classificationProblemData != null) {
+        var model = CreateClassificationModel(classificationProblemData,
+                                              nTrees, r, m, seed, out rmsError, out oobRmsError,
+                                              out relClassError, out oobRelClassError);
+        randomForest = model.randomForest;
+      }
+    }
+
+    public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, AllowedInputVariables, rows);
+      AssertInputMatrix(inputData);
 
       int n = inputData.GetLength(0);
@@ -100,16 +142,17 @@
           x[column] = inputData[row, column];
         }
-        alglib.dfprocess(randomForest, x, ref y);
+        alglib.dfprocess(RandomForest, x, ref y);
         yield return y[0];
       }
     }
 
-    public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
-      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+    public IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, AllowedInputVariables, rows);
+      AssertInputMatrix(inputData);
 
       int n = inputData.GetLength(0);
       int columns = inputData.GetLength(1);
       double[] x = new double[columns];
-      double[] y = new double[randomForest.innerobj.nclasses];
+      double[] y = new double[RandomForest.innerobj.nclasses];
 
       for (int row = 0; row < n; row++) {
@@ -144,18 +187,110 @@
     }
 
-    #region events
-    public event EventHandler Changed;
-    private void OnChanged(EventArgs e) {
-      var handlers = Changed;
-      if (handlers != null)
-        handlers(this, e);
-    }
-    #endregion
-
-    #region persistence
+    public static RandomForestModel CreateRegressionModel(IRegressionProblemData problemData, int nTrees, double r, double m, int seed,
+      out double rmsError, out double outOfBagRmsError, out double avgRelError, out double outOfBagAvgRelError) {
+      return CreateRegressionModel(problemData, problemData.TrainingIndices, nTrees, r, m, seed, out rmsError, out avgRelError, out outOfBagAvgRelError, out outOfBagRmsError);
+    }
+
+    public static RandomForestModel CreateRegressionModel(IRegressionProblemData problemData, IEnumerable<int> trainingIndices, int nTrees, double r, double m, int seed,
+      out double rmsError, out double outOfBagRmsError, out double avgRelError, out double outOfBagAvgRelError) {
+      var variables = problemData.AllowedInputVariables.Concat(new string[] { problemData.TargetVariable });
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(problemData.Dataset, variables, trainingIndices);
+
+      alglib.dfreport rep;
+      var dForest = CreateRandomForestModel(seed, inputMatrix, nTrees, r, m, 1, out rep);
+
+      rmsError = rep.rmserror;
+      avgRelError = rep.avgrelerror;
+      outOfBagAvgRelError = rep.oobavgrelerror;
+      outOfBagRmsError = rep.oobrmserror;
+
+      return new RandomForestModel(dForest, seed, problemData, nTrees, r, m);
+    }
+
+    public static RandomForestModel CreateClassificationModel(IClassificationProblemData problemData, int nTrees, double r, double m, int seed,
+      out double rmsError, out double outOfBagRmsError, out double relClassificationError, out double outOfBagRelClassificationError) {
+      return CreateClassificationModel(problemData, problemData.TrainingIndices, nTrees, r, m, seed, out rmsError, out outOfBagRmsError, out relClassificationError, out outOfBagRelClassificationError);
+    }
+
+    public static RandomForestModel CreateClassificationModel(IClassificationProblemData problemData, IEnumerable<int> trainingIndices, int nTrees, double r, double m, int seed,
+      out double rmsError, out double outOfBagRmsError, out double relClassificationError, out double outOfBagRelClassificationError) {
+
+      var variables = problemData.AllowedInputVariables.Concat(new string[] { problemData.TargetVariable });
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(problemData.Dataset, variables, trainingIndices);
+
+      var classValues = problemData.ClassValues.ToArray();
+      int nClasses = classValues.Length;
+
+      // map original class values to values [0..nClasses-1]
+      var classIndices = new Dictionary<double, double>();
+      for (int i = 0; i < nClasses; i++) {
+        classIndices[classValues[i]] = i;
+      }
+
+      int nRows = inputMatrix.GetLength(0);
+      int nColumns = inputMatrix.GetLength(1);
+      for (int row = 0; row < nRows; row++) {
+        inputMatrix[row, nColumns - 1] = classIndices[inputMatrix[row, nColumns - 1]];
+      }
+
+      alglib.dfreport rep;
+      var dForest = CreateRandomForestModel(seed, inputMatrix, nTrees, r, m, nClasses, out rep);
+
+      rmsError = rep.rmserror;
+      outOfBagRmsError = rep.oobrmserror;
+      relClassificationError = rep.relclserror;
+      outOfBagRelClassificationError = rep.oobrelclserror;
+
+      return new RandomForestModel(dForest, seed, problemData, nTrees, r, m, classValues);
+    }
+
+    private static alglib.decisionforest CreateRandomForestModel(int seed, double[,] inputMatrix, int nTrees, double r, double m, int nClasses, out alglib.dfreport rep) {
+      AssertParameters(r, m);
+      AssertInputMatrix(inputMatrix);
+
+      int info = 0;
+      alglib.math.rndobject = new System.Random(seed);
+      var dForest = new alglib.decisionforest();
+      rep = new alglib.dfreport();
+      int nRows = inputMatrix.GetLength(0);
+      int nColumns = inputMatrix.GetLength(1);
+      int sampleSize = Math.Max((int)Math.Round(r * nRows), 1);
+      int nFeatures = Math.Max((int)Math.Round(m * (nColumns - 1)), 1);
+
+      alglib.dforest.dfbuildinternal(inputMatrix, nRows, nColumns - 1, nClasses, nTrees, sampleSize, nFeatures, alglib.dforest.dfusestrongsplits + alglib.dforest.dfuseevs, ref info, dForest.innerobj, rep.innerobj);
+      if (info != 1) throw new ArgumentException("Error in calculation of random forest model");
+      return dForest;
+    }
+
+    private static void AssertParameters(double r, double m) {
+      if (r <= 0 || r > 1) throw new ArgumentException("The R parameter for random forest modeling must be between 0 and 1.");
+      if (m <= 0 || m > 1) throw new ArgumentException("The M parameter for random forest modeling must be between 0 and 1.");
+    }
+
+    private static void AssertInputMatrix(double[,] inputMatrix) {
+      if (inputMatrix.Cast<double>().Any(x => Double.IsNaN(x) || Double.IsInfinity(x)))
+        throw new NotSupportedException("Random forest modeling does not support NaN or infinity values in the input dataset.");
+    }
+
+    #region persistence for backwards compatibility
+    // when the originalTrainingData is null this means the model was loaded from an old file 
+    // therefore, we cannot use the new persistence mechanism because the original data is not available anymore
+    // in such cases we still store the compete model
+    private bool IsCompatibilityLoaded { get { return originalTrainingData == null; } }
+
+    private string[] allowedInputVariables;
+    [Storable(Name = "allowedInputVariables")]
+    private string[] AllowedInputVariables {
+      get {
+        if (IsCompatibilityLoaded) return allowedInputVariables;
+        else return originalTrainingData.AllowedInputVariables.ToArray();
+      }
+      set { allowedInputVariables = value; }
+    }
     [Storable]
     private int RandomForestBufSize {
       get {
-        return randomForest.innerobj.bufsize;
+        if (IsCompatibilityLoaded) return randomForest.innerobj.bufsize;
+        else return 0;
       }
       set {
@@ -166,5 +301,6 @@
     private int RandomForestNClasses {
       get {
-        return randomForest.innerobj.nclasses;
+        if (IsCompatibilityLoaded) return randomForest.innerobj.nclasses;
+        else return 0;
       }
       set {
@@ -175,5 +311,6 @@
     private int RandomForestNTrees {
       get {
-        return randomForest.innerobj.ntrees;
+        if (IsCompatibilityLoaded) return randomForest.innerobj.ntrees;
+        else return 0;
       }
       set {
@@ -184,5 +321,6 @@
     private int RandomForestNVars {
       get {
-        return randomForest.innerobj.nvars;
+        if (IsCompatibilityLoaded) return randomForest.innerobj.nvars;
+        else return 0;
       }
       set {
@@ -193,5 +331,6 @@
     private double[] RandomForestTrees {
       get {
-        return randomForest.innerobj.trees;
+        if (IsCompatibilityLoaded) return randomForest.innerobj.trees;
+        else return new double[] { };
       }
       set {
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegression.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegression.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegression.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -20,7 +20,4 @@
 #endregion
 
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -36,5 +33,5 @@
   /// </summary>
   [Item("Random Forest Regression", "Random forest regression data analysis algorithm (wrapper for ALGLIB).")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 120)]
   [StorableClass]
   public sealed class RandomForestRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
@@ -130,34 +127,6 @@
     public static IRegressionSolution CreateRandomForestRegressionSolution(IRegressionProblemData problemData, int nTrees, double r, double m, int seed,
       out double rmsError, out double avgRelError, out double outOfBagRmsError, out double outOfBagAvgRelError) {
-      if (r <= 0 || r > 1) throw new ArgumentException("The R parameter in the random forest regression must be between 0 and 1.");
-      if (m <= 0 || m > 1) throw new ArgumentException("The M parameter in the random forest regression must be between 0 and 1.");
-
-      alglib.math.rndobject = new System.Random(seed);
-
-      Dataset dataset = problemData.Dataset;
-      string targetVariable = problemData.TargetVariable;
-      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
-      IEnumerable<int> rows = problemData.TrainingIndices;
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
-      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
-        throw new NotSupportedException("Random forest regression does not support NaN or infinity values in the input dataset.");
-
-      int info = 0;
-      alglib.decisionforest dForest = new alglib.decisionforest();
-      alglib.dfreport rep = new alglib.dfreport(); ;
-      int nRows = inputMatrix.GetLength(0);
-      int nColumns = inputMatrix.GetLength(1);
-      int sampleSize = Math.Max((int)Math.Round(r * nRows), 1);
-      int nFeatures = Math.Max((int)Math.Round(m * (nColumns - 1)), 1);
-
-      alglib.dforest.dfbuildinternal(inputMatrix, nRows, nColumns - 1, 1, nTrees, sampleSize, nFeatures, alglib.dforest.dfusestrongsplits + alglib.dforest.dfuseevs, ref info, dForest.innerobj, rep.innerobj);
-      if (info != 1) throw new ArgumentException("Error in calculation of random forest regression solution");
-
-      rmsError = rep.rmserror;
-      avgRelError = rep.avgrelerror;
-      outOfBagAvgRelError = rep.oobavgrelerror;
-      outOfBagRmsError = rep.oobrmserror;
-
-      return new RandomForestRegressionSolution((IRegressionProblemData)problemData.Clone(), new RandomForestModel(dForest, targetVariable, allowedInputVariables));
+      var model = RandomForestModel.CreateRegressionModel(problemData, nTrees, r, m, seed, out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError);
+      return new RandomForestRegressionSolution((IRegressionProblemData)problemData.Clone(), model);
     }
     #endregion
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegressionSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegressionSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegressionSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestUtil.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestUtil.cs	(revision 13085)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestUtil.cs	(revision 13085)
@@ -0,0 +1,358 @@
+﻿#region License Information
+
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Linq.Expressions;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Random;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("RFParameter", "A random forest parameter collection")]
+  [StorableClass]
+  public class RFParameter : ParameterCollection {
+    public RFParameter() {
+      base.Add(new FixedValueParameter<IntValue>("N", "The number of random forest trees", new IntValue(50)));
+      base.Add(new FixedValueParameter<DoubleValue>("M", "The ratio of features that will be used in the construction of individual trees (0<m<=1)", new DoubleValue(0.1)));
+      base.Add(new FixedValueParameter<DoubleValue>("R", "The ratio of the training set that will be used in the construction of individual trees (0<r<=1)", new DoubleValue(0.1)));
+    }
+
+    [StorableConstructor]
+    protected RFParameter(bool deserializing)
+      : base(deserializing) {
+    }
+
+    protected RFParameter(RFParameter original, Cloner cloner)
+      : base(original, cloner) {
+      this.N = original.N;
+      this.R = original.R;
+      this.M = original.M;
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RFParameter(this, cloner);
+    }
+
+    private IFixedValueParameter<IntValue> NParameter {
+      get { return (IFixedValueParameter<IntValue>)base["N"]; }
+    }
+
+    private IFixedValueParameter<DoubleValue> RParameter {
+      get { return (IFixedValueParameter<DoubleValue>)base["R"]; }
+    }
+
+    private IFixedValueParameter<DoubleValue> MParameter {
+      get { return (IFixedValueParameter<DoubleValue>)base["M"]; }
+    }
+
+    public int N {
+      get { return NParameter.Value.Value; }
+      set { NParameter.Value.Value = value; }
+    }
+
+    public double R {
+      get { return RParameter.Value.Value; }
+      set { RParameter.Value.Value = value; }
+    }
+
+    public double M {
+      get { return MParameter.Value.Value; }
+      set { MParameter.Value.Value = value; }
+    }
+  }
+
+  public static class RandomForestUtil {
+    private static void CrossValidate(IRegressionProblemData problemData, Tuple<IEnumerable<int>, IEnumerable<int>>[] partitions, int nTrees, double r, double m, int seed, out double avgTestMse) {
+      avgTestMse = 0;
+      var ds = problemData.Dataset;
+      var targetVariable = GetTargetVariableName(problemData);
+      foreach (var tuple in partitions) {
+        double rmsError, avgRelError, outOfBagAvgRelError, outOfBagRmsError;
+        var trainingRandomForestPartition = tuple.Item1;
+        var testRandomForestPartition = tuple.Item2;
+        var model = RandomForestModel.CreateRegressionModel(problemData, trainingRandomForestPartition, nTrees, r, m, seed, out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError);
+        var estimatedValues = model.GetEstimatedValues(ds, testRandomForestPartition);
+        var targetValues = ds.GetDoubleValues(targetVariable, testRandomForestPartition);
+        OnlineCalculatorError calculatorError;
+        double mse = OnlineMeanSquaredErrorCalculator.Calculate(estimatedValues, targetValues, out calculatorError);
+        if (calculatorError != OnlineCalculatorError.None)
+          mse = double.NaN;
+        avgTestMse += mse;
+      }
+      avgTestMse /= partitions.Length;
+    }
+
+    private static void CrossValidate(IClassificationProblemData problemData, Tuple<IEnumerable<int>, IEnumerable<int>>[] partitions, int nTrees, double r, double m, int seed, out double avgTestAccuracy) {
+      avgTestAccuracy = 0;
+      var ds = problemData.Dataset;
+      var targetVariable = GetTargetVariableName(problemData);
+      foreach (var tuple in partitions) {
+        double rmsError, avgRelError, outOfBagAvgRelError, outOfBagRmsError;
+        var trainingRandomForestPartition = tuple.Item1;
+        var testRandomForestPartition = tuple.Item2;
+        var model = RandomForestModel.CreateClassificationModel(problemData, trainingRandomForestPartition, nTrees, r, m, seed, out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError);
+        var estimatedValues = model.GetEstimatedClassValues(ds, testRandomForestPartition);
+        var targetValues = ds.GetDoubleValues(targetVariable, testRandomForestPartition);
+        OnlineCalculatorError calculatorError;
+        double accuracy = OnlineAccuracyCalculator.Calculate(estimatedValues, targetValues, out calculatorError);
+        if (calculatorError != OnlineCalculatorError.None)
+          accuracy = double.NaN;
+        avgTestAccuracy += accuracy;
+      }
+      avgTestAccuracy /= partitions.Length;
+    }
+
+    /// <summary>
+    /// Grid search without crossvalidation (since for random forests the out-of-bag estimate is unbiased)
+    /// </summary>
+    /// <param name="problemData">The regression problem data</param>
+    /// <param name="parameterRanges">The ranges for each parameter in the grid search</param>
+    /// <param name="seed">The random seed (required by the random forest model)</param>
+    /// <param name="maxDegreeOfParallelism">The maximum allowed number of threads (to parallelize the grid search)</param>
+    public static RFParameter GridSearch(IRegressionProblemData problemData, Dictionary<string, IEnumerable<double>> parameterRanges, int seed = 12345, int maxDegreeOfParallelism = 1) {
+      var setters = parameterRanges.Keys.Select(GenerateSetter).ToList();
+      var crossProduct = parameterRanges.Values.CartesianProduct();
+      double bestOutOfBagRmsError = double.MaxValue;
+      RFParameter bestParameters = new RFParameter();
+
+      var locker = new object();
+      Parallel.ForEach(crossProduct, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, parameterCombination => {
+        var parameterValues = parameterCombination.ToList();
+        var parameters = new RFParameter();
+        for (int i = 0; i < setters.Count; ++i) { setters[i](parameters, parameterValues[i]); }
+        double rmsError, outOfBagRmsError, avgRelError, outOfBagAvgRelError;
+        RandomForestModel.CreateRegressionModel(problemData, problemData.TrainingIndices, parameters.N, parameters.R, parameters.M, seed, out rmsError, out outOfBagRmsError, out avgRelError, out outOfBagAvgRelError);
+
+        lock (locker) {
+          if (bestOutOfBagRmsError > outOfBagRmsError) {
+            bestOutOfBagRmsError = outOfBagRmsError;
+            bestParameters = (RFParameter)parameters.Clone();
+          }
+        }
+      });
+      return bestParameters;
+    }
+
+    /// <summary>
+    /// Grid search without crossvalidation (since for random forests the out-of-bag estimate is unbiased)
+    /// </summary>
+    /// <param name="problemData">The classification problem data</param>
+    /// <param name="parameterRanges">The ranges for each parameter in the grid search</param>
+    /// <param name="seed">The random seed (required by the random forest model)</param>
+    /// <param name="maxDegreeOfParallelism">The maximum allowed number of threads (to parallelize the grid search)</param>
+    public static RFParameter GridSearch(IClassificationProblemData problemData, Dictionary<string, IEnumerable<double>> parameterRanges, int seed = 12345, int maxDegreeOfParallelism = 1) {
+      var setters = parameterRanges.Keys.Select(GenerateSetter).ToList();
+      var crossProduct = parameterRanges.Values.CartesianProduct();
+
+      double bestOutOfBagRmsError = double.MaxValue;
+      RFParameter bestParameters = new RFParameter();
+
+      var locker = new object();
+      Parallel.ForEach(crossProduct, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, parameterCombination => {
+        var parameterValues = parameterCombination.ToList();
+        var parameters = new RFParameter();
+        for (int i = 0; i < setters.Count; ++i) { setters[i](parameters, parameterValues[i]); }
+        double rmsError, outOfBagRmsError, avgRelError, outOfBagAvgRelError;
+        RandomForestModel.CreateClassificationModel(problemData, problemData.TrainingIndices, parameters.N, parameters.R, parameters.M, seed,
+                                                                out rmsError, out outOfBagRmsError, out avgRelError, out outOfBagAvgRelError);
+
+        lock (locker) {
+          if (bestOutOfBagRmsError > outOfBagRmsError) {
+            bestOutOfBagRmsError = outOfBagRmsError;
+            bestParameters = (RFParameter)parameters.Clone();
+          }
+        }
+      });
+      return bestParameters;
+    }
+
+    /// <summary>
+    /// Grid search with crossvalidation
+    /// </summary>
+    /// <param name="problemData">The regression problem data</param>
+    /// <param name="numberOfFolds">The number of folds for crossvalidation</param>
+    /// <param name="shuffleFolds">Specifies whether the folds should be shuffled</param>
+    /// <param name="parameterRanges">The ranges for each parameter in the grid search</param>
+    /// <param name="seed">The random seed (required by the random forest model)</param>
+    /// <param name="maxDegreeOfParallelism">The maximum allowed number of threads (to parallelize the grid search)</param>
+    /// <returns>The best parameter values found by the grid search</returns>
+    public static RFParameter GridSearch(IRegressionProblemData problemData, int numberOfFolds, bool shuffleFolds, Dictionary<string, IEnumerable<double>> parameterRanges, int seed = 12345, int maxDegreeOfParallelism = 1) {
+      DoubleValue mse = new DoubleValue(Double.MaxValue);
+      RFParameter bestParameter = new RFParameter();
+
+      var setters = parameterRanges.Keys.Select(GenerateSetter).ToList();
+      var partitions = GenerateRandomForestPartitions(problemData, numberOfFolds);
+      var crossProduct = parameterRanges.Values.CartesianProduct();
+
+      var locker = new object();
+      Parallel.ForEach(crossProduct, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, parameterCombination => {
+        var parameterValues = parameterCombination.ToList();
+        double testMSE;
+        var parameters = new RFParameter();
+        for (int i = 0; i < setters.Count; ++i) {
+          setters[i](parameters, parameterValues[i]);
+        }
+        CrossValidate(problemData, partitions, parameters.N, parameters.R, parameters.M, seed, out testMSE);
+
+        lock (locker) {
+          if (testMSE < mse.Value) {
+            mse.Value = testMSE;
+            bestParameter = (RFParameter)parameters.Clone();
+          }
+        }
+      });
+      return bestParameter;
+    }
+
+    /// <summary>
+    /// Grid search with crossvalidation
+    /// </summary>
+    /// <param name="problemData">The classification problem data</param>
+    /// <param name="numberOfFolds">The number of folds for crossvalidation</param>
+    /// <param name="shuffleFolds">Specifies whether the folds should be shuffled</param>
+    /// <param name="parameterRanges">The ranges for each parameter in the grid search</param>
+    /// <param name="seed">The random seed (for shuffling)</param>
+    /// <param name="maxDegreeOfParallelism">The maximum allowed number of threads (to parallelize the grid search)</param>
+    public static RFParameter GridSearch(IClassificationProblemData problemData, int numberOfFolds, bool shuffleFolds, Dictionary<string, IEnumerable<double>> parameterRanges, int seed = 12345, int maxDegreeOfParallelism = 1) {
+      DoubleValue accuracy = new DoubleValue(0);
+      RFParameter bestParameter = new RFParameter();
+
+      var setters = parameterRanges.Keys.Select(GenerateSetter).ToList();
+      var crossProduct = parameterRanges.Values.CartesianProduct();
+      var partitions = GenerateRandomForestPartitions(problemData, numberOfFolds, shuffleFolds);
+
+      var locker = new object();
+      Parallel.ForEach(crossProduct, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, parameterCombination => {
+        var parameterValues = parameterCombination.ToList();
+        double testAccuracy;
+        var parameters = new RFParameter();
+        for (int i = 0; i < setters.Count; ++i) {
+          setters[i](parameters, parameterValues[i]);
+        }
+        CrossValidate(problemData, partitions, parameters.N, parameters.R, parameters.M, seed, out testAccuracy);
+
+        lock (locker) {
+          if (testAccuracy > accuracy.Value) {
+            accuracy.Value = testAccuracy;
+            bestParameter = (RFParameter)parameters.Clone();
+          }
+        }
+      });
+      return bestParameter;
+    }
+
+    private static Tuple<IEnumerable<int>, IEnumerable<int>>[] GenerateRandomForestPartitions(IDataAnalysisProblemData problemData, int numberOfFolds, bool shuffleFolds = false) {
+      var folds = GenerateFolds(problemData, numberOfFolds, shuffleFolds).ToList();
+      var partitions = new Tuple<IEnumerable<int>, IEnumerable<int>>[numberOfFolds];
+
+      for (int i = 0; i < numberOfFolds; ++i) {
+        int p = i; // avoid "access to modified closure" warning
+        var trainingRows = folds.SelectMany((par, j) => j != p ? par : Enumerable.Empty<int>());
+        var testRows = folds[i];
+        partitions[i] = new Tuple<IEnumerable<int>, IEnumerable<int>>(trainingRows, testRows);
+      }
+      return partitions;
+    }
+
+    public static IEnumerable<IEnumerable<int>> GenerateFolds(IDataAnalysisProblemData problemData, int numberOfFolds, bool shuffleFolds = false) {
+      var random = new MersenneTwister((uint)Environment.TickCount);
+      if (problemData is IRegressionProblemData) {
+        var trainingIndices = shuffleFolds ? problemData.TrainingIndices.OrderBy(x => random.Next()) : problemData.TrainingIndices;
+        return GenerateFolds(trainingIndices, problemData.TrainingPartition.Size, numberOfFolds);
+      }
+      if (problemData is IClassificationProblemData) {
+        // when shuffle is enabled do stratified folds generation, some folds may have zero elements 
+        // otherwise, generate folds normally
+        return shuffleFolds ? GenerateFoldsStratified(problemData as IClassificationProblemData, numberOfFolds, random) : GenerateFolds(problemData.TrainingIndices, problemData.TrainingPartition.Size, numberOfFolds);
+      }
+      throw new ArgumentException("Problem data is neither regression or classification problem data.");
+    }
+
+    /// <summary>
+    /// Stratified fold generation from classification data. Stratification means that we ensure the same distribution of class labels for each fold.
+    /// The samples are grouped by class label and each group is split into @numberOfFolds parts. The final folds are formed from the joining of 
+    /// the corresponding parts from each class label.
+    /// </summary>
+    /// <param name="problemData">The classification problem data.</param>
+    /// <param name="numberOfFolds">The number of folds in which to split the data.</param>
+    /// <param name="random">The random generator used to shuffle the folds.</param>
+    /// <returns>An enumerable sequece of folds, where a fold is represented by a sequence of row indices.</returns>
+    private static IEnumerable<IEnumerable<int>> GenerateFoldsStratified(IClassificationProblemData problemData, int numberOfFolds, IRandom random) {
+      var values = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+      var valuesIndices = problemData.TrainingIndices.Zip(values, (i, v) => new { Index = i, Value = v }).ToList();
+      IEnumerable<IEnumerable<IEnumerable<int>>> foldsByClass = valuesIndices.GroupBy(x => x.Value, x => x.Index).Select(g => GenerateFolds(g, g.Count(), numberOfFolds));
+      var enumerators = foldsByClass.Select(f => f.GetEnumerator()).ToList();
+      while (enumerators.All(e => e.MoveNext())) {
+        yield return enumerators.SelectMany(e => e.Current).OrderBy(x => random.Next()).ToList();
+      }
+    }
+
+    private static IEnumerable<IEnumerable<T>> GenerateFolds<T>(IEnumerable<T> values, int valuesCount, int numberOfFolds) {
+      // if number of folds is greater than the number of values, some empty folds will be returned
+      if (valuesCount < numberOfFolds) {
+        for (int i = 0; i < numberOfFolds; ++i)
+          yield return i < valuesCount ? values.Skip(i).Take(1) : Enumerable.Empty<T>();
+      } else {
+        int f = valuesCount / numberOfFolds, r = valuesCount % numberOfFolds; // number of folds rounded to integer and remainder
+        int start = 0, end = f;
+        for (int i = 0; i < numberOfFolds; ++i) {
+          if (r > 0) {
+            ++end;
+            --r;
+          }
+          yield return values.Skip(start).Take(end - start);
+          start = end;
+          end += f;
+        }
+      }
+    }
+
+    private static Action<RFParameter, double> GenerateSetter(string field) {
+      var targetExp = Expression.Parameter(typeof(RFParameter));
+      var valueExp = Expression.Parameter(typeof(double));
+      var fieldExp = Expression.Property(targetExp, field);
+      var assignExp = Expression.Assign(fieldExp, Expression.Convert(valueExp, fieldExp.Type));
+      var setter = Expression.Lambda<Action<RFParameter, double>>(assignExp, targetExp, valueExp).Compile();
+      return setter;
+    }
+
+    private static string GetTargetVariableName(IDataAnalysisProblemData problemData) {
+      var regressionProblemData = problemData as IRegressionProblemData;
+      var classificationProblemData = problemData as IClassificationProblemData;
+
+      if (regressionProblemData != null)
+        return regressionProblemData.TargetVariable;
+      if (classificationProblemData != null)
+        return classificationProblemData.TargetVariable;
+
+      throw new ArgumentException("Problem data is neither regression or classification problem data.");
+    }
+  }
+}
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassification.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassification.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassification.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -37,5 +37,5 @@
   /// </summary>
   [Item("Support Vector Classification", "Support vector machine classification data analysis algorithm (wrapper for libSVM).")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 110)]
   [StorableClass]
   public sealed class SupportVectorClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
@@ -46,4 +46,5 @@
     private const string GammaParameterName = "Gamma";
     private const string DegreeParameterName = "Degree";
+    private const string CreateSolutionParameterName = "CreateSolution";
 
     #region parameter properties
@@ -65,4 +66,7 @@
     public IValueParameter<IntValue> DegreeParameter {
       get { return (IValueParameter<IntValue>)Parameters[DegreeParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> CreateSolutionParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
     }
     #endregion
@@ -87,4 +91,8 @@
     public IntValue Degree {
       get { return DegreeParameter.Value; }
+    }
+    public bool CreateSolution {
+      get { return CreateSolutionParameter.Value.Value; }
+      set { CreateSolutionParameter.Value.Value = value; }
     }
     #endregion
@@ -112,10 +120,19 @@
       Parameters.Add(new ValueParameter<DoubleValue>(GammaParameterName, "The value of the gamma parameter in the kernel function.", new DoubleValue(1.0)));
       Parameters.Add(new ValueParameter<IntValue>(DegreeParameterName, "The degree parameter for the polynomial kernel function.", new IntValue(3)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+      Parameters[CreateSolutionParameterName].Hidden = true;
     }
     [StorableHook(HookType.AfterDeserialization)]
     private void AfterDeserialization() {
       #region backwards compatibility (change with 3.4)
-      if (!Parameters.ContainsKey(DegreeParameterName))
-        Parameters.Add(new ValueParameter<IntValue>(DegreeParameterName, "The degree parameter for the polynomial kernel function.", new IntValue(3)));
+      if (!Parameters.ContainsKey(DegreeParameterName)) {
+        Parameters.Add(new ValueParameter<IntValue>(DegreeParameterName,
+          "The degree parameter for the polynomial kernel function.", new IntValue(3)));
+      }
+      if (!Parameters.ContainsKey(CreateSolutionParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName,
+          "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+        Parameters[CreateSolutionParameterName].Hidden = true;
+      }
       #endregion
     }
@@ -129,37 +146,84 @@
       IClassificationProblemData problemData = Problem.ProblemData;
       IEnumerable<string> selectedInputVariables = problemData.AllowedInputVariables;
-      double trainingAccuracy, testAccuracy;
       int nSv;
-      var solution = CreateSupportVectorClassificationSolution(problemData, selectedInputVariables,
-        SvmType.Value, KernelType.Value, Cost.Value, Nu.Value, Gamma.Value, Degree.Value,
+      ISupportVectorMachineModel model;
+
+      Run(problemData, selectedInputVariables, GetSvmType(SvmType.Value), GetKernelType(KernelType.Value), Cost.Value, Nu.Value, Gamma.Value, Degree.Value, out model, out nSv);
+
+      if (CreateSolution) {
+        var solution = new SupportVectorClassificationSolution((SupportVectorMachineModel)model, (IClassificationProblemData)problemData.Clone());
+        Results.Add(new Result("Support vector classification solution", "The support vector classification solution.",
+          solution));
+      }
+
+      {
+        // calculate classification metrics
+        // calculate regression model metrics 
+        var ds = problemData.Dataset;
+        var trainRows = problemData.TrainingIndices;
+        var testRows = problemData.TestIndices;
+        var yTrain = ds.GetDoubleValues(problemData.TargetVariable, trainRows);
+        var yTest = ds.GetDoubleValues(problemData.TargetVariable, testRows);
+        var yPredTrain = model.GetEstimatedClassValues(ds, trainRows);
+        var yPredTest = model.GetEstimatedClassValues(ds, testRows);
+
+        OnlineCalculatorError error;
+        var trainAccuracy = OnlineAccuracyCalculator.Calculate(yPredTrain, yTrain, out error);
+        if (error != OnlineCalculatorError.None) trainAccuracy = double.MaxValue;
+        var testAccuracy = OnlineAccuracyCalculator.Calculate(yPredTest, yTest, out error);
+        if (error != OnlineCalculatorError.None) testAccuracy = double.MaxValue;
+
+        Results.Add(new Result("Accuracy (training)", "The mean of squared errors of the SVR solution on the training partition.", new DoubleValue(trainAccuracy)));
+        Results.Add(new Result("Accuracy (test)", "The mean of squared errors of the SVR solution on the test partition.", new DoubleValue(testAccuracy)));
+
+        Results.Add(new Result("Number of support vectors", "The number of support vectors of the SVR solution.",
+          new IntValue(nSv)));
+      }
+    }
+
+    public static SupportVectorClassificationSolution CreateSupportVectorClassificationSolution(IClassificationProblemData problemData, IEnumerable<string> allowedInputVariables,
+      string svmType, string kernelType, double cost, double nu, double gamma, int degree, out double trainingAccuracy, out double testAccuracy, out int nSv) {
+      return CreateSupportVectorClassificationSolution(problemData, allowedInputVariables, GetSvmType(svmType), GetKernelType(kernelType), cost, nu, gamma, degree,
         out trainingAccuracy, out testAccuracy, out nSv);
-
-      Results.Add(new Result("Support vector classification solution", "The support vector classification solution.", solution));
-      Results.Add(new Result("Training accuracy", "The accuracy of the SVR solution on the training partition.", new DoubleValue(trainingAccuracy)));
-      Results.Add(new Result("Test accuracy", "The accuracy of the SVR solution on the test partition.", new DoubleValue(testAccuracy)));
-      Results.Add(new Result("Number of support vectors", "The number of support vectors of the SVR solution.", new IntValue(nSv)));
-    }
-
+    }
+
+    // BackwardsCompatibility3.4
+    #region Backwards compatible code, remove with 3.5
     public static SupportVectorClassificationSolution CreateSupportVectorClassificationSolution(IClassificationProblemData problemData, IEnumerable<string> allowedInputVariables,
-      string svmType, string kernelType, double cost, double nu, double gamma, int degree,
-      out double trainingAccuracy, out double testAccuracy, out int nSv) {
-      Dataset dataset = problemData.Dataset;
+      int svmType, int kernelType, double cost, double nu, double gamma, int degree, out double trainingAccuracy, out double testAccuracy, out int nSv) {
+
+      ISupportVectorMachineModel model;
+      Run(problemData, allowedInputVariables, svmType, kernelType, cost, nu, gamma, degree, out model, out nSv);
+      var solution = new SupportVectorClassificationSolution((SupportVectorMachineModel)model, (IClassificationProblemData)problemData.Clone());
+
+      trainingAccuracy = solution.TrainingAccuracy;
+      testAccuracy = solution.TestAccuracy;
+
+      return solution;
+    }
+
+    #endregion
+
+    public static void Run(IClassificationProblemData problemData, IEnumerable<string> allowedInputVariables,
+      int svmType, int kernelType, double cost, double nu, double gamma, int degree,
+      out ISupportVectorMachineModel model, out int nSv) {
+      var dataset = problemData.Dataset;
       string targetVariable = problemData.TargetVariable;
       IEnumerable<int> rows = problemData.TrainingIndices;
 
       //extract SVM parameters from scope and set them
-      svm_parameter parameter = new svm_parameter();
-      parameter.svm_type = GetSvmType(svmType);
-      parameter.kernel_type = GetKernelType(kernelType);
-      parameter.C = cost;
-      parameter.nu = nu;
-      parameter.gamma = gamma;
-      parameter.cache_size = 500;
-      parameter.probability = 0;
-      parameter.eps = 0.001;
-      parameter.degree = degree;
-      parameter.shrinking = 1;
-      parameter.coef0 = 0;
-
+      svm_parameter parameter = new svm_parameter {
+        svm_type = svmType,
+        kernel_type = kernelType,
+        C = cost,
+        nu = nu,
+        gamma = gamma,
+        cache_size = 500,
+        probability = 0,
+        eps = 0.001,
+        degree = degree,
+        shrinking = 1,
+        coef0 = 0
+      };
 
       var weightLabels = new List<int>();
@@ -178,17 +242,11 @@
       parameter.weight = weights.ToArray();
 
-
       svm_problem problem = SupportVectorMachineUtil.CreateSvmProblem(dataset, targetVariable, allowedInputVariables, rows);
       RangeTransform rangeTransform = RangeTransform.Compute(problem);
       svm_problem scaledProblem = rangeTransform.Scale(problem);
       var svmModel = svm.svm_train(scaledProblem, parameter);
-      var model = new SupportVectorMachineModel(svmModel, rangeTransform, targetVariable, allowedInputVariables, problemData.ClassValues);
-      var solution = new SupportVectorClassificationSolution(model, (IClassificationProblemData)problemData.Clone());
-
       nSv = svmModel.SV.Length;
-      trainingAccuracy = solution.TrainingAccuracy;
-      testAccuracy = solution.TestAccuracy;
-
-      return solution;
+
+      model = new SupportVectorMachineModel(svmModel, rangeTransform, targetVariable, allowedInputVariables, problemData.ClassValues);
     }
 
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassificationSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassificationSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassificationSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineModel.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineModel.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineModel.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -120,5 +120,5 @@
 
     #region IRegressionModel Members
-    public IEnumerable<double> GetEstimatedValues(Dataset dataset, IEnumerable<int> rows) {
+    public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
       return GetEstimatedValuesHelper(dataset, rows);
     }
@@ -132,5 +132,5 @@
 
     #region IClassificationModel Members
-    public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
+    public IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
       if (classValues == null) throw new NotSupportedException();
       // return the original class value instead of the predicted value of the model
@@ -159,5 +159,5 @@
     }
     #endregion
-    private IEnumerable<double> GetEstimatedValuesHelper(Dataset dataset, IEnumerable<int> rows) {
+    private IEnumerable<double> GetEstimatedValuesHelper(IDataset dataset, IEnumerable<int> rows) {
       // calculate predictions for the currently requested rows
       svm_problem problem = SupportVectorMachineUtil.CreateSvmProblem(dataset, targetVariable, allowedInputVariables, rows);
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineUtil.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineUtil.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineUtil.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -20,7 +20,14 @@
 #endregion
 
+using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Linq.Expressions;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
 using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Random;
 using LibSVM;
 
@@ -33,15 +40,12 @@
     /// <param name="rowIndices">The rows of the dataset that should be contained in the resulting SVM-problem</param>
     /// <returns>A problem data type that can be used to train a support vector machine.</returns>
-    public static svm_problem CreateSvmProblem(Dataset dataset, string targetVariable, IEnumerable<string> inputVariables, IEnumerable<int> rowIndices) {
-      double[] targetVector =
-        dataset.GetDoubleValues(targetVariable, rowIndices).ToArray();
-
+    public static svm_problem CreateSvmProblem(IDataset dataset, string targetVariable, IEnumerable<string> inputVariables, IEnumerable<int> rowIndices) {
+      double[] targetVector = dataset.GetDoubleValues(targetVariable, rowIndices).ToArray();
       svm_node[][] nodes = new svm_node[targetVector.Length][];
-      List<svm_node> tempRow;
       int maxNodeIndex = 0;
       int svmProblemRowIndex = 0;
       List<string> inputVariablesList = inputVariables.ToList();
       foreach (int row in rowIndices) {
-        tempRow = new List<svm_node>();
+        List<svm_node> tempRow = new List<svm_node>();
         int colIndex = 1; // make sure the smallest node index for SVM = 1
         foreach (var inputVariable in inputVariablesList) {
@@ -50,5 +54,6 @@
           // => don't add NaN values in the dataset to the sparse SVM matrix representation
           if (!double.IsNaN(value)) {
-            tempRow.Add(new svm_node() { index = colIndex, value = value }); // nodes must be sorted in ascending ordered by column index
+            tempRow.Add(new svm_node() { index = colIndex, value = value });
+            // nodes must be sorted in ascending ordered by column index
             if (colIndex > maxNodeIndex) maxNodeIndex = colIndex;
           }
@@ -57,6 +62,168 @@
         nodes[svmProblemRowIndex++] = tempRow.ToArray();
       }
-
-      return new svm_problem() { l = targetVector.Length, y = targetVector, x = nodes };
+      return new svm_problem { l = targetVector.Length, y = targetVector, x = nodes };
+    }
+
+    /// <summary>
+    /// Instantiate and return a svm_parameter object with default values.
+    /// </summary>
+    /// <returns>A svm_parameter object with default values</returns>
+    public static svm_parameter DefaultParameters() {
+      svm_parameter parameter = new svm_parameter();
+      parameter.svm_type = svm_parameter.NU_SVR;
+      parameter.kernel_type = svm_parameter.RBF;
+      parameter.C = 1;
+      parameter.nu = 0.5;
+      parameter.gamma = 1;
+      parameter.p = 1;
+      parameter.cache_size = 500;
+      parameter.probability = 0;
+      parameter.eps = 0.001;
+      parameter.degree = 3;
+      parameter.shrinking = 1;
+      parameter.coef0 = 0;
+
+      return parameter;
+    }
+
+    public static double CrossValidate(IDataAnalysisProblemData problemData, svm_parameter parameters, int numberOfFolds, bool shuffleFolds = true) {
+      var partitions = GenerateSvmPartitions(problemData, numberOfFolds, shuffleFolds);
+      return CalculateCrossValidationPartitions(partitions, parameters);
+    }
+
+    public static svm_parameter GridSearch(out double cvMse, IDataAnalysisProblemData problemData, Dictionary<string, IEnumerable<double>> parameterRanges, int numberOfFolds, bool shuffleFolds = true, int maxDegreeOfParallelism = 1) {
+      DoubleValue mse = new DoubleValue(Double.MaxValue);
+      var bestParam = DefaultParameters();
+      var crossProduct = parameterRanges.Values.CartesianProduct();
+      var setters = parameterRanges.Keys.Select(GenerateSetter).ToList();
+      var partitions = GenerateSvmPartitions(problemData, numberOfFolds, shuffleFolds);
+
+      var locker = new object(); // for thread synchronization
+      Parallel.ForEach(crossProduct, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
+      parameterCombination => {
+        var parameters = DefaultParameters();
+        var parameterValues = parameterCombination.ToList();
+        for (int i = 0; i < parameterValues.Count; ++i)
+          setters[i](parameters, parameterValues[i]);
+
+        double testMse = CalculateCrossValidationPartitions(partitions, parameters);
+        if (!double.IsNaN(testMse)) {
+          lock (locker) {
+            if (testMse < mse.Value) {
+              mse.Value = testMse;
+              bestParam = (svm_parameter)parameters.Clone();
+            }
+          }
+        }
+      });
+      cvMse = mse.Value;
+      return bestParam;
+    }
+
+    private static double CalculateCrossValidationPartitions(Tuple<svm_problem, svm_problem>[] partitions, svm_parameter parameters) {
+      double avgTestMse = 0;
+      var calc = new OnlineMeanSquaredErrorCalculator();
+      foreach (Tuple<svm_problem, svm_problem> tuple in partitions) {
+        var trainingSvmProblem = tuple.Item1;
+        var testSvmProblem = tuple.Item2;
+        var model = svm.svm_train(trainingSvmProblem, parameters);
+        calc.Reset();
+        for (int i = 0; i < testSvmProblem.l; ++i)
+          calc.Add(testSvmProblem.y[i], svm.svm_predict(model, testSvmProblem.x[i]));
+        double mse = calc.ErrorState == OnlineCalculatorError.None ? calc.MeanSquaredError : double.NaN;
+        avgTestMse += mse;
+      }
+      avgTestMse /= partitions.Length;
+      return avgTestMse;
+    }
+
+    private static Tuple<svm_problem, svm_problem>[] GenerateSvmPartitions(IDataAnalysisProblemData problemData, int numberOfFolds, bool shuffleFolds = true) {
+      var folds = GenerateFolds(problemData, numberOfFolds, shuffleFolds).ToList();
+      var targetVariable = GetTargetVariableName(problemData);
+      var partitions = new Tuple<svm_problem, svm_problem>[numberOfFolds];
+      for (int i = 0; i < numberOfFolds; ++i) {
+        int p = i; // avoid "access to modified closure" warning below
+        var trainingRows = folds.SelectMany((par, j) => j != p ? par : Enumerable.Empty<int>());
+        var testRows = folds[i];
+        var trainingSvmProblem = CreateSvmProblem(problemData.Dataset, targetVariable, problemData.AllowedInputVariables, trainingRows);
+        var rangeTransform = RangeTransform.Compute(trainingSvmProblem);
+        var testSvmProblem = rangeTransform.Scale(CreateSvmProblem(problemData.Dataset, targetVariable, problemData.AllowedInputVariables, testRows));
+        partitions[i] = new Tuple<svm_problem, svm_problem>(rangeTransform.Scale(trainingSvmProblem), testSvmProblem);
+      }
+      return partitions;
+    }
+
+    public static IEnumerable<IEnumerable<int>> GenerateFolds(IDataAnalysisProblemData problemData, int numberOfFolds, bool shuffleFolds = true) {
+      var random = new MersenneTwister((uint)Environment.TickCount);
+      if (problemData is IRegressionProblemData) {
+        var trainingIndices = shuffleFolds ? problemData.TrainingIndices.OrderBy(x => random.Next()) : problemData.TrainingIndices;
+        return GenerateFolds(trainingIndices, problemData.TrainingPartition.Size, numberOfFolds);
+      }
+      if (problemData is IClassificationProblemData) {
+        // when shuffle is enabled do stratified folds generation, some folds may have zero elements 
+        // otherwise, generate folds normally
+        return shuffleFolds ? GenerateFoldsStratified(problemData as IClassificationProblemData, numberOfFolds, random) : GenerateFolds(problemData.TrainingIndices, problemData.TrainingPartition.Size, numberOfFolds);
+      }
+      throw new ArgumentException("Problem data is neither regression or classification problem data.");
+    }
+
+    /// <summary>
+    /// Stratified fold generation from classification data. Stratification means that we ensure the same distribution of class labels for each fold.
+    /// The samples are grouped by class label and each group is split into @numberOfFolds parts. The final folds are formed from the joining of 
+    /// the corresponding parts from each class label.
+    /// </summary>
+    /// <param name="problemData">The classification problem data.</param>
+    /// <param name="numberOfFolds">The number of folds in which to split the data.</param>
+    /// <param name="random">The random generator used to shuffle the folds.</param>
+    /// <returns>An enumerable sequece of folds, where a fold is represented by a sequence of row indices.</returns>
+    private static IEnumerable<IEnumerable<int>> GenerateFoldsStratified(IClassificationProblemData problemData, int numberOfFolds, IRandom random) {
+      var values = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+      var valuesIndices = problemData.TrainingIndices.Zip(values, (i, v) => new { Index = i, Value = v }).ToList();
+      IEnumerable<IEnumerable<IEnumerable<int>>> foldsByClass = valuesIndices.GroupBy(x => x.Value, x => x.Index).Select(g => GenerateFolds(g, g.Count(), numberOfFolds));
+      var enumerators = foldsByClass.Select(f => f.GetEnumerator()).ToList();
+      while (enumerators.All(e => e.MoveNext())) {
+        yield return enumerators.SelectMany(e => e.Current).OrderBy(x => random.Next()).ToList();
+      }
+    }
+
+    private static IEnumerable<IEnumerable<T>> GenerateFolds<T>(IEnumerable<T> values, int valuesCount, int numberOfFolds) {
+      // if number of folds is greater than the number of values, some empty folds will be returned
+      if (valuesCount < numberOfFolds) {
+        for (int i = 0; i < numberOfFolds; ++i)
+          yield return i < valuesCount ? values.Skip(i).Take(1) : Enumerable.Empty<T>();
+      } else {
+        int f = valuesCount / numberOfFolds, r = valuesCount % numberOfFolds; // number of folds rounded to integer and remainder
+        int start = 0, end = f;
+        for (int i = 0; i < numberOfFolds; ++i) {
+          if (r > 0) {
+            ++end;
+            --r;
+          }
+          yield return values.Skip(start).Take(end - start);
+          start = end;
+          end += f;
+        }
+      }
+    }
+
+    private static Action<svm_parameter, double> GenerateSetter(string fieldName) {
+      var targetExp = Expression.Parameter(typeof(svm_parameter));
+      var valueExp = Expression.Parameter(typeof(double));
+      var fieldExp = Expression.Field(targetExp, fieldName);
+      var assignExp = Expression.Assign(fieldExp, Expression.Convert(valueExp, fieldExp.Type));
+      var setter = Expression.Lambda<Action<svm_parameter, double>>(assignExp, targetExp, valueExp).Compile();
+      return setter;
+    }
+
+    private static string GetTargetVariableName(IDataAnalysisProblemData problemData) {
+      var regressionProblemData = problemData as IRegressionProblemData;
+      var classificationProblemData = problemData as IClassificationProblemData;
+
+      if (regressionProblemData != null)
+        return regressionProblemData.TargetVariable;
+      if (classificationProblemData != null)
+        return classificationProblemData.TargetVariable;
+
+      throw new ArgumentException("Problem data is neither regression or classification problem data.");
     }
   }
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegression.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegression.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegression.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -37,5 +37,5 @@
   /// </summary>
   [Item("Support Vector Regression", "Support vector machine regression data analysis algorithm (wrapper for libSVM).")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 110)]
   [StorableClass]
   public sealed class SupportVectorRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
@@ -47,4 +47,5 @@
     private const string EpsilonParameterName = "Epsilon";
     private const string DegreeParameterName = "Degree";
+    private const string CreateSolutionParameterName = "CreateSolution";
 
     #region parameter properties
@@ -69,4 +70,7 @@
     public IValueParameter<IntValue> DegreeParameter {
       get { return (IValueParameter<IntValue>)Parameters[DegreeParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> CreateSolutionParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
     }
     #endregion
@@ -94,4 +98,8 @@
     public IntValue Degree {
       get { return DegreeParameter.Value; }
+    }
+    public bool CreateSolution {
+      get { return CreateSolutionParameter.Value.Value; }
+      set { CreateSolutionParameter.Value.Value = value; }
     }
     #endregion
@@ -120,10 +128,19 @@
       Parameters.Add(new ValueParameter<DoubleValue>(EpsilonParameterName, "The value of the epsilon parameter for epsilon-SVR.", new DoubleValue(0.1)));
       Parameters.Add(new ValueParameter<IntValue>(DegreeParameterName, "The degree parameter for the polynomial kernel function.", new IntValue(3)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+      Parameters[CreateSolutionParameterName].Hidden = true;
     }
     [StorableHook(HookType.AfterDeserialization)]
     private void AfterDeserialization() {
       #region backwards compatibility (change with 3.4)
-      if (!Parameters.ContainsKey(DegreeParameterName))
-        Parameters.Add(new ValueParameter<IntValue>(DegreeParameterName, "The degree parameter for the polynomial kernel function.", new IntValue(3)));
+
+      if (!Parameters.ContainsKey(DegreeParameterName)) {
+        Parameters.Add(new ValueParameter<IntValue>(DegreeParameterName,
+          "The degree parameter for the polynomial kernel function.", new IntValue(3)));
+      }
+      if (!Parameters.ContainsKey(CreateSolutionParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+        Parameters[CreateSolutionParameterName].Hidden = true;
+      }
       #endregion
     }
@@ -137,39 +154,94 @@
       IRegressionProblemData problemData = Problem.ProblemData;
       IEnumerable<string> selectedInputVariables = problemData.AllowedInputVariables;
-      double trainR2, testR2;
       int nSv;
-      var solution = CreateSupportVectorRegressionSolution(problemData, selectedInputVariables, SvmType.Value,
-        KernelType.Value, Cost.Value, Nu.Value, Gamma.Value, Epsilon.Value, Degree.Value,
-        out trainR2, out testR2, out nSv);
-
-      Results.Add(new Result("Support vector regression solution", "The support vector regression solution.", solution));
-      Results.Add(new Result("Training R²", "The Pearson's R² of the SVR solution on the training partition.", new DoubleValue(trainR2)));
-      Results.Add(new Result("Test R²", "The Pearson's R² of the SVR solution on the test partition.", new DoubleValue(testR2)));
+      ISupportVectorMachineModel model;
+      Run(problemData, selectedInputVariables, SvmType.Value, KernelType.Value, Cost.Value, Nu.Value, Gamma.Value, Epsilon.Value, Degree.Value, out model, out nSv);
+
+      if (CreateSolution) {
+        var solution = new SupportVectorRegressionSolution((SupportVectorMachineModel)model, (IRegressionProblemData)problemData.Clone());
+        Results.Add(new Result("Support vector regression solution", "The support vector regression solution.", solution));
+      }
+
       Results.Add(new Result("Number of support vectors", "The number of support vectors of the SVR solution.", new IntValue(nSv)));
-    }
-
-    public static SupportVectorRegressionSolution CreateSupportVectorRegressionSolution(IRegressionProblemData problemData, IEnumerable<string> allowedInputVariables,
+
+
+      {
+        // calculate regression model metrics 
+        var ds = problemData.Dataset;
+        var trainRows = problemData.TrainingIndices;
+        var testRows = problemData.TestIndices;
+        var yTrain = ds.GetDoubleValues(problemData.TargetVariable, trainRows);
+        var yTest = ds.GetDoubleValues(problemData.TargetVariable, testRows);
+        var yPredTrain = model.GetEstimatedValues(ds, trainRows).ToArray();
+        var yPredTest = model.GetEstimatedValues(ds, testRows).ToArray();
+
+        OnlineCalculatorError error;
+        var trainMse = OnlineMeanSquaredErrorCalculator.Calculate(yPredTrain, yTrain, out error);
+        if (error != OnlineCalculatorError.None) trainMse = double.MaxValue;
+        var testMse = OnlineMeanSquaredErrorCalculator.Calculate(yPredTest, yTest, out error);
+        if (error != OnlineCalculatorError.None) testMse = double.MaxValue;
+
+        Results.Add(new Result("Mean squared error (training)", "The mean of squared errors of the SVR solution on the training partition.", new DoubleValue(trainMse)));
+        Results.Add(new Result("Mean squared error (test)", "The mean of squared errors of the SVR solution on the test partition.", new DoubleValue(testMse)));
+
+
+        var trainMae = OnlineMeanAbsoluteErrorCalculator.Calculate(yPredTrain, yTrain, out error);
+        if (error != OnlineCalculatorError.None) trainMae = double.MaxValue;
+        var testMae = OnlineMeanAbsoluteErrorCalculator.Calculate(yPredTest, yTest, out error);
+        if (error != OnlineCalculatorError.None) testMae = double.MaxValue;
+
+        Results.Add(new Result("Mean absolute error (training)", "The mean of absolute errors of the SVR solution on the training partition.", new DoubleValue(trainMae)));
+        Results.Add(new Result("Mean absolute error (test)", "The mean of absolute errors of the SVR solution on the test partition.", new DoubleValue(testMae)));
+
+
+        var trainRelErr = OnlineMeanAbsolutePercentageErrorCalculator.Calculate(yPredTrain, yTrain, out error);
+        if (error != OnlineCalculatorError.None) trainRelErr = double.MaxValue;
+        var testRelErr = OnlineMeanAbsolutePercentageErrorCalculator.Calculate(yPredTest, yTest, out error);
+        if (error != OnlineCalculatorError.None) testRelErr = double.MaxValue;
+
+        Results.Add(new Result("Average relative error (training)", "The mean of relative errors of the SVR solution on the training partition.", new DoubleValue(trainRelErr)));
+        Results.Add(new Result("Average relative error (test)", "The mean of relative errors of the SVR solution on the test partition.", new DoubleValue(testRelErr)));
+      }
+    }
+
+    // BackwardsCompatibility3.4
+    #region Backwards compatible code, remove with 3.5
+    // for compatibility with old API
+    public static SupportVectorRegressionSolution CreateSupportVectorRegressionSolution(
+      IRegressionProblemData problemData, IEnumerable<string> allowedInputVariables,
       string svmType, string kernelType, double cost, double nu, double gamma, double epsilon, int degree,
       out double trainingR2, out double testR2, out int nSv) {
-      Dataset dataset = problemData.Dataset;
+      ISupportVectorMachineModel model;
+      Run(problemData, allowedInputVariables, svmType, kernelType, cost, nu, gamma, epsilon, degree, out model, out nSv);
+
+      var solution = new SupportVectorRegressionSolution((SupportVectorMachineModel)model, (IRegressionProblemData)problemData.Clone());
+      trainingR2 = solution.TrainingRSquared;
+      testR2 = solution.TestRSquared;
+      return solution;
+    }
+    #endregion
+
+    public static void Run(IRegressionProblemData problemData, IEnumerable<string> allowedInputVariables,
+      string svmType, string kernelType, double cost, double nu, double gamma, double epsilon, int degree,
+      out ISupportVectorMachineModel model, out int nSv) {
+      var dataset = problemData.Dataset;
       string targetVariable = problemData.TargetVariable;
       IEnumerable<int> rows = problemData.TrainingIndices;
 
       //extract SVM parameters from scope and set them
-      svm_parameter parameter = new svm_parameter();
-      parameter.svm_type = GetSvmType(svmType);
-      parameter.kernel_type = GetKernelType(kernelType);
-      parameter.C = cost;
-      parameter.nu = nu;
-      parameter.gamma = gamma;
-      parameter.p = epsilon;
-      parameter.cache_size = 500;
-      parameter.probability = 0;
-      parameter.eps = 0.001;
-      parameter.degree = degree;
-      parameter.shrinking = 1;
-      parameter.coef0 = 0;
-
-
+      svm_parameter parameter = new svm_parameter {
+        svm_type = GetSvmType(svmType),
+        kernel_type = GetKernelType(kernelType),
+        C = cost,
+        nu = nu,
+        gamma = gamma,
+        p = epsilon,
+        cache_size = 500,
+        probability = 0,
+        eps = 0.001,
+        degree = degree,
+        shrinking = 1,
+        coef0 = 0
+      };
 
       svm_problem problem = SupportVectorMachineUtil.CreateSvmProblem(dataset, targetVariable, allowedInputVariables, rows);
@@ -178,9 +250,6 @@
       var svmModel = svm.svm_train(scaledProblem, parameter);
       nSv = svmModel.SV.Length;
-      var model = new SupportVectorMachineModel(svmModel, rangeTransform, targetVariable, allowedInputVariables);
-      var solution = new SupportVectorRegressionSolution(model, (IRegressionProblemData)problemData.Clone());
-      trainingR2 = solution.TrainingRSquared;
-      testR2 = solution.TestRSquared;
-      return solution;
+
+      model = new SupportVectorMachineModel(svmModel, rangeTransform, targetVariable, allowedInputVariables);
     }
 
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegressionSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegressionSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegressionSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/TimeSeries/AutoregressiveModeling.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/TimeSeries/AutoregressiveModeling.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/TimeSeries/AutoregressiveModeling.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -35,5 +35,5 @@
 namespace HeuristicLab.Algorithms.DataAnalysis.TimeSeries {
   [Item("Autoregressive Modeling", "Timeseries modeling algorithm that creates AR-N models.")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 130)]
   [StorableClass]
   public class AutoregressiveModeling : FixedDataAnalysisAlgorithm<ITimeSeriesPrognosisProblem> {
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClustering.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClustering.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClustering.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -36,5 +36,5 @@
   /// </summary>
   [Item("k-Means", "The k-Means clustering algorithm (wrapper for ALGLIB).")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 10)]
   [StorableClass]
   public sealed class KMeansClustering : FixedDataAnalysisAlgorithm<IClusteringProblem> {
@@ -83,5 +83,5 @@
 
     public static KMeansClusteringSolution CreateKMeansSolution(IClusteringProblemData problemData, int k, int restarts) {
-      Dataset dataset = problemData.Dataset;
+      var dataset = problemData.Dataset;
       IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
       IEnumerable<int> rows = problemData.TrainingIndices;
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringModel.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringModel.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringModel.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -81,5 +81,5 @@
 
 
-    public IEnumerable<int> GetClusterValues(Dataset dataset, IEnumerable<int> rows) {
+    public IEnumerable<int> GetClusterValues(IDataset dataset, IEnumerable<int> rows) {
       return KMeansClusteringUtil.FindClosestCenters(centers, dataset, allowedInputVariables, rows);
     }
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringSolution.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringSolution.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringSolution.cs	(revision 13085)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringUtil.cs
===================================================================
--- branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringUtil.cs	(revision 13084)
+++ branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringUtil.cs	(revision 13085)
@@ -1,5 +1,5 @@
 ﻿#region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -27,5 +27,5 @@
 namespace HeuristicLab.Algorithms.DataAnalysis {
   public static class KMeansClusteringUtil {
-    public static IEnumerable<int> FindClosestCenters(IEnumerable<double[]> centers, Dataset dataset, IEnumerable<string> allowedInputVariables, IEnumerable<int> rows) {
+    public static IEnumerable<int> FindClosestCenters(IEnumerable<double[]> centers, IDataset dataset, IEnumerable<string> allowedInputVariables, IEnumerable<int> rows) {
       int nRows = rows.Count();
       int nCols = allowedInputVariables.Count();
@@ -58,5 +58,5 @@
     }
 
-    public static double CalculateIntraClusterSumOfSquares(KMeansClusteringModel model, Dataset dataset, IEnumerable<int> rows) {
+    public static double CalculateIntraClusterSumOfSquares(KMeansClusteringModel model, IDataset dataset, IEnumerable<int> rows) {
       List<int> clusterValues = model.GetClusterValues(dataset, rows).ToList();
       List<string> allowedInputVariables = model.AllowedInputVariables.ToList();
