Index: branches/gteufl/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassification.cs
===================================================================
--- branches/gteufl/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassification.cs	(revision 10225)
+++ branches/gteufl/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassification.cs	(revision 12969)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -20,7 +20,4 @@
 #endregion
 
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -36,5 +33,5 @@
   /// </summary>
   [Item("Random Forest Classification", "Random forest classification data analysis algorithm (wrapper for ALGLIB).")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 120)]
   [StorableClass]
   public sealed class RandomForestClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
@@ -131,45 +128,6 @@
     public static IClassificationSolution CreateRandomForestClassificationSolution(IClassificationProblemData problemData, int nTrees, double r, double m, int seed,
       out double rmsError, out double relClassificationError, out double outOfBagRmsError, out double outOfBagRelClassificationError) {
-      if (r <= 0 || r > 1) throw new ArgumentException("The R parameter in the random forest regression must be between 0 and 1.");
-      if (m <= 0 || m > 1) throw new ArgumentException("The M parameter in the random forest regression must be between 0 and 1.");
-
-      alglib.math.rndobject = new System.Random(seed);
-
-      Dataset dataset = problemData.Dataset;
-      string targetVariable = problemData.TargetVariable;
-      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
-      IEnumerable<int> rows = problemData.TrainingIndices;
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
-      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
-        throw new NotSupportedException("Random forest classification does not support NaN or infinity values in the input dataset.");
-
-      int info = 0;
-      alglib.decisionforest dForest = new alglib.decisionforest();
-      alglib.dfreport rep = new alglib.dfreport(); ;
-      int nRows = inputMatrix.GetLength(0);
-      int nColumns = inputMatrix.GetLength(1);
-      int sampleSize = Math.Max((int)Math.Round(r * nRows), 1);
-      int nFeatures = Math.Max((int)Math.Round(m * (nColumns - 1)), 1);
-
-
-      double[] classValues = problemData.ClassValues.ToArray();
-      int nClasses = problemData.Classes;
-      // map original class values to values [0..nClasses-1]
-      Dictionary<double, double> classIndices = new Dictionary<double, double>();
-      for (int i = 0; i < nClasses; i++) {
-        classIndices[classValues[i]] = i;
-      }
-      for (int row = 0; row < nRows; row++) {
-        inputMatrix[row, nColumns - 1] = classIndices[inputMatrix[row, nColumns - 1]];
-      }
-      // execute random forest algorithm      
-      alglib.dforest.dfbuildinternal(inputMatrix, nRows, nColumns - 1, nClasses, nTrees, sampleSize, nFeatures, alglib.dforest.dfusestrongsplits + alglib.dforest.dfuseevs, ref info, dForest.innerobj, rep.innerobj);
-      if (info != 1) throw new ArgumentException("Error in calculation of random forest classification solution");
-
-      rmsError = rep.rmserror;
-      outOfBagRmsError = rep.oobrmserror;
-      relClassificationError = rep.relclserror;
-      outOfBagRelClassificationError = rep.oobrelclserror;
-      return new RandomForestClassificationSolution((IClassificationProblemData)problemData.Clone(), new RandomForestModel(dForest, targetVariable, allowedInputVariables, classValues));
+      var model = RandomForestModel.CreateClassificationModel(problemData, nTrees, r, m, seed, out rmsError, out relClassificationError, out outOfBagRmsError, out outOfBagRelClassificationError);
+      return new RandomForestClassificationSolution((IClassificationProblemData)problemData.Clone(), model);
     }
     #endregion
Index: branches/gteufl/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassificationSolution.cs
===================================================================
--- branches/gteufl/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassificationSolution.cs	(revision 10225)
+++ branches/gteufl/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassificationSolution.cs	(revision 12969)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/gteufl/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs
===================================================================
--- branches/gteufl/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs	(revision 10225)
+++ branches/gteufl/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs	(revision 12969)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -35,28 +35,35 @@
   [Item("RandomForestModel", "Represents a random forest for regression and classification.")]
   public sealed class RandomForestModel : NamedItem, IRandomForestModel {
-
+    // not persisted
     private alglib.decisionforest randomForest;
-    public alglib.decisionforest RandomForest {
-      get { return randomForest; }
-      set {
-        if (value != randomForest) {
-          if (value == null) throw new ArgumentNullException();
-          randomForest = value;
-          OnChanged(EventArgs.Empty);
-        }
-      }
-    }
-
-    [Storable]
-    private string targetVariable;
-    [Storable]
-    private string[] allowedInputVariables;
+    private alglib.decisionforest RandomForest {
+      get {
+        // recalculate lazily
+        if (randomForest.innerobj.trees == null || randomForest.innerobj.trees.Length == 0) RecalculateModel();
+        return randomForest;
+      }
+    }
+
+    // instead of storing the data of the model itself
+    // we instead only store data necessary to recalculate the same model lazily on demand
+    [Storable]
+    private int seed;
+    [Storable]
+    private IDataAnalysisProblemData originalTrainingData;
     [Storable]
     private double[] classValues;
+    [Storable]
+    private int nTrees;
+    [Storable]
+    private double r;
+    [Storable]
+    private double m;
+
+
     [StorableConstructor]
     private RandomForestModel(bool deserializing)
       : base(deserializing) {
-      if (deserializing)
-        randomForest = new alglib.decisionforest();
+      // for backwards compatibility (loading old solutions)
+      randomForest = new alglib.decisionforest();
     }
     private RandomForestModel(RandomForestModel original, Cloner cloner)
@@ -67,19 +74,36 @@
       randomForest.innerobj.ntrees = original.randomForest.innerobj.ntrees;
       randomForest.innerobj.nvars = original.randomForest.innerobj.nvars;
-      randomForest.innerobj.trees = (double[])original.randomForest.innerobj.trees.Clone();
-      targetVariable = original.targetVariable;
-      allowedInputVariables = (string[])original.allowedInputVariables.Clone();
-      if (original.classValues != null)
-        this.classValues = (double[])original.classValues.Clone();
-    }
-    public RandomForestModel(alglib.decisionforest randomForest, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues = null)
+      // we assume that the trees array (double[]) is immutable in alglib
+      randomForest.innerobj.trees = original.randomForest.innerobj.trees;
+
+      // allowedInputVariables is immutable so we don't need to clone
+      allowedInputVariables = original.allowedInputVariables;
+
+      // clone data which is necessary to rebuild the model
+      this.seed = original.seed;
+      this.originalTrainingData = cloner.Clone(original.originalTrainingData);
+      // classvalues is immutable so we don't need to clone
+      this.classValues = original.classValues;
+      this.nTrees = original.nTrees;
+      this.r = original.r;
+      this.m = original.m;
+    }
+
+    // random forest models can only be created through the static factory methods CreateRegressionModel and CreateClassificationModel
+    private RandomForestModel(alglib.decisionforest randomForest,
+      int seed, IDataAnalysisProblemData originalTrainingData,
+      int nTrees, double r, double m, double[] classValues = null)
       : base() {
       this.name = ItemName;
       this.description = ItemDescription;
+      // the model itself
       this.randomForest = randomForest;
-      this.targetVariable = targetVariable;
-      this.allowedInputVariables = allowedInputVariables.ToArray();
-      if (classValues != null)
-        this.classValues = (double[])classValues.Clone();
+      // data which is necessary for recalculation of the model
+      this.seed = seed;
+      this.originalTrainingData = (IDataAnalysisProblemData)originalTrainingData.Clone();
+      this.classValues = classValues;
+      this.nTrees = nTrees;
+      this.r = r;
+      this.m = m;
     }
 
@@ -88,6 +112,24 @@
     }
 
-    public IEnumerable<double> GetEstimatedValues(Dataset dataset, IEnumerable<int> rows) {
-      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+    private void RecalculateModel() {
+      double rmsError, oobRmsError, relClassError, oobRelClassError;
+      var regressionProblemData = originalTrainingData as IRegressionProblemData;
+      var classificationProblemData = originalTrainingData as IClassificationProblemData;
+      if (regressionProblemData != null) {
+        var model = CreateRegressionModel(regressionProblemData,
+                                              nTrees, r, m, seed, out rmsError, out oobRmsError,
+                                              out relClassError, out oobRelClassError);
+        randomForest = model.randomForest;
+      } else if (classificationProblemData != null) {
+        var model = CreateClassificationModel(classificationProblemData,
+                                              nTrees, r, m, seed, out rmsError, out oobRmsError,
+                                              out relClassError, out oobRelClassError);
+        randomForest = model.randomForest;
+      }
+    }
+
+    public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, AllowedInputVariables, rows);
+      AssertInputMatrix(inputData);
 
       int n = inputData.GetLength(0);
@@ -100,16 +142,17 @@
           x[column] = inputData[row, column];
         }
-        alglib.dfprocess(randomForest, x, ref y);
+        alglib.dfprocess(RandomForest, x, ref y);
         yield return y[0];
       }
     }
 
-    public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
-      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+    public IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, AllowedInputVariables, rows);
+      AssertInputMatrix(inputData);
 
       int n = inputData.GetLength(0);
       int columns = inputData.GetLength(1);
       double[] x = new double[columns];
-      double[] y = new double[randomForest.innerobj.nclasses];
+      double[] y = new double[RandomForest.innerobj.nclasses];
 
       for (int row = 0; row < n; row++) {
@@ -144,18 +187,110 @@
     }
 
-    #region events
-    public event EventHandler Changed;
-    private void OnChanged(EventArgs e) {
-      var handlers = Changed;
-      if (handlers != null)
-        handlers(this, e);
-    }
-    #endregion
-
-    #region persistence
+    public static RandomForestModel CreateRegressionModel(IRegressionProblemData problemData, int nTrees, double r, double m, int seed,
+      out double rmsError, out double outOfBagRmsError, out double avgRelError, out double outOfBagAvgRelError) {
+      return CreateRegressionModel(problemData, problemData.TrainingIndices, nTrees, r, m, seed, out rmsError, out avgRelError, out outOfBagAvgRelError, out outOfBagRmsError);
+    }
+
+    public static RandomForestModel CreateRegressionModel(IRegressionProblemData problemData, IEnumerable<int> trainingIndices, int nTrees, double r, double m, int seed,
+      out double rmsError, out double outOfBagRmsError, out double avgRelError, out double outOfBagAvgRelError) {
+      var variables = problemData.AllowedInputVariables.Concat(new string[] { problemData.TargetVariable });
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(problemData.Dataset, variables, trainingIndices);
+
+      alglib.dfreport rep;
+      var dForest = CreateRandomForestModel(seed, inputMatrix, nTrees, r, m, 1, out rep);
+
+      rmsError = rep.rmserror;
+      avgRelError = rep.avgrelerror;
+      outOfBagAvgRelError = rep.oobavgrelerror;
+      outOfBagRmsError = rep.oobrmserror;
+
+      return new RandomForestModel(dForest, seed, problemData, nTrees, r, m);
+    }
+
+    public static RandomForestModel CreateClassificationModel(IClassificationProblemData problemData, int nTrees, double r, double m, int seed,
+      out double rmsError, out double outOfBagRmsError, out double relClassificationError, out double outOfBagRelClassificationError) {
+      return CreateClassificationModel(problemData, problemData.TrainingIndices, nTrees, r, m, seed, out rmsError, out outOfBagRmsError, out relClassificationError, out outOfBagRelClassificationError);
+    }
+
+    public static RandomForestModel CreateClassificationModel(IClassificationProblemData problemData, IEnumerable<int> trainingIndices, int nTrees, double r, double m, int seed,
+      out double rmsError, out double outOfBagRmsError, out double relClassificationError, out double outOfBagRelClassificationError) {
+
+      var variables = problemData.AllowedInputVariables.Concat(new string[] { problemData.TargetVariable });
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(problemData.Dataset, variables, trainingIndices);
+
+      var classValues = problemData.ClassValues.ToArray();
+      int nClasses = classValues.Length;
+
+      // map original class values to values [0..nClasses-1]
+      var classIndices = new Dictionary<double, double>();
+      for (int i = 0; i < nClasses; i++) {
+        classIndices[classValues[i]] = i;
+      }
+
+      int nRows = inputMatrix.GetLength(0);
+      int nColumns = inputMatrix.GetLength(1);
+      for (int row = 0; row < nRows; row++) {
+        inputMatrix[row, nColumns - 1] = classIndices[inputMatrix[row, nColumns - 1]];
+      }
+
+      alglib.dfreport rep;
+      var dForest = CreateRandomForestModel(seed, inputMatrix, nTrees, r, m, nClasses, out rep);
+
+      rmsError = rep.rmserror;
+      outOfBagRmsError = rep.oobrmserror;
+      relClassificationError = rep.relclserror;
+      outOfBagRelClassificationError = rep.oobrelclserror;
+
+      return new RandomForestModel(dForest, seed, problemData, nTrees, r, m, classValues);
+    }
+
+    private static alglib.decisionforest CreateRandomForestModel(int seed, double[,] inputMatrix, int nTrees, double r, double m, int nClasses, out alglib.dfreport rep) {
+      AssertParameters(r, m);
+      AssertInputMatrix(inputMatrix);
+
+      int info = 0;
+      alglib.math.rndobject = new System.Random(seed);
+      var dForest = new alglib.decisionforest();
+      rep = new alglib.dfreport();
+      int nRows = inputMatrix.GetLength(0);
+      int nColumns = inputMatrix.GetLength(1);
+      int sampleSize = Math.Max((int)Math.Round(r * nRows), 1);
+      int nFeatures = Math.Max((int)Math.Round(m * (nColumns - 1)), 1);
+
+      alglib.dforest.dfbuildinternal(inputMatrix, nRows, nColumns - 1, nClasses, nTrees, sampleSize, nFeatures, alglib.dforest.dfusestrongsplits + alglib.dforest.dfuseevs, ref info, dForest.innerobj, rep.innerobj);
+      if (info != 1) throw new ArgumentException("Error in calculation of random forest model");
+      return dForest;
+    }
+
+    private static void AssertParameters(double r, double m) {
+      if (r <= 0 || r > 1) throw new ArgumentException("The R parameter for random forest modeling must be between 0 and 1.");
+      if (m <= 0 || m > 1) throw new ArgumentException("The M parameter for random forest modeling must be between 0 and 1.");
+    }
+
+    private static void AssertInputMatrix(double[,] inputMatrix) {
+      if (inputMatrix.Cast<double>().Any(x => Double.IsNaN(x) || Double.IsInfinity(x)))
+        throw new NotSupportedException("Random forest modeling does not support NaN or infinity values in the input dataset.");
+    }
+
+    #region persistence for backwards compatibility
+    // when the originalTrainingData is null this means the model was loaded from an old file 
+    // therefore, we cannot use the new persistence mechanism because the original data is not available anymore
+    // in such cases we still store the compete model
+    private bool IsCompatibilityLoaded { get { return originalTrainingData == null; } }
+
+    private string[] allowedInputVariables;
+    [Storable(Name = "allowedInputVariables")]
+    private string[] AllowedInputVariables {
+      get {
+        if (IsCompatibilityLoaded) return allowedInputVariables;
+        else return originalTrainingData.AllowedInputVariables.ToArray();
+      }
+      set { allowedInputVariables = value; }
+    }
     [Storable]
     private int RandomForestBufSize {
       get {
-        return randomForest.innerobj.bufsize;
+        if (IsCompatibilityLoaded) return randomForest.innerobj.bufsize;
+        else return 0;
       }
       set {
@@ -166,5 +301,6 @@
     private int RandomForestNClasses {
       get {
-        return randomForest.innerobj.nclasses;
+        if (IsCompatibilityLoaded) return randomForest.innerobj.nclasses;
+        else return 0;
       }
       set {
@@ -175,5 +311,6 @@
     private int RandomForestNTrees {
       get {
-        return randomForest.innerobj.ntrees;
+        if (IsCompatibilityLoaded) return randomForest.innerobj.ntrees;
+        else return 0;
       }
       set {
@@ -184,5 +321,6 @@
     private int RandomForestNVars {
       get {
-        return randomForest.innerobj.nvars;
+        if (IsCompatibilityLoaded) return randomForest.innerobj.nvars;
+        else return 0;
       }
       set {
@@ -193,5 +331,6 @@
     private double[] RandomForestTrees {
       get {
-        return randomForest.innerobj.trees;
+        if (IsCompatibilityLoaded) return randomForest.innerobj.trees;
+        else return new double[] { };
       }
       set {
Index: branches/gteufl/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegression.cs
===================================================================
--- branches/gteufl/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegression.cs	(revision 10225)
+++ branches/gteufl/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegression.cs	(revision 12969)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -20,7 +20,4 @@
 #endregion
 
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -36,5 +33,5 @@
   /// </summary>
   [Item("Random Forest Regression", "Random forest regression data analysis algorithm (wrapper for ALGLIB).")]
-  [Creatable("Data Analysis")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 120)]
   [StorableClass]
   public sealed class RandomForestRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
@@ -130,34 +127,6 @@
     public static IRegressionSolution CreateRandomForestRegressionSolution(IRegressionProblemData problemData, int nTrees, double r, double m, int seed,
       out double rmsError, out double avgRelError, out double outOfBagRmsError, out double outOfBagAvgRelError) {
-      if (r <= 0 || r > 1) throw new ArgumentException("The R parameter in the random forest regression must be between 0 and 1.");
-      if (m <= 0 || m > 1) throw new ArgumentException("The M parameter in the random forest regression must be between 0 and 1.");
-
-      alglib.math.rndobject = new System.Random(seed);
-
-      Dataset dataset = problemData.Dataset;
-      string targetVariable = problemData.TargetVariable;
-      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
-      IEnumerable<int> rows = problemData.TrainingIndices;
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
-      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
-        throw new NotSupportedException("Random forest regression does not support NaN or infinity values in the input dataset.");
-
-      int info = 0;
-      alglib.decisionforest dForest = new alglib.decisionforest();
-      alglib.dfreport rep = new alglib.dfreport(); ;
-      int nRows = inputMatrix.GetLength(0);
-      int nColumns = inputMatrix.GetLength(1);
-      int sampleSize = Math.Max((int)Math.Round(r * nRows), 1);
-      int nFeatures = Math.Max((int)Math.Round(m * (nColumns - 1)), 1);
-
-      alglib.dforest.dfbuildinternal(inputMatrix, nRows, nColumns - 1, 1, nTrees, sampleSize, nFeatures, alglib.dforest.dfusestrongsplits + alglib.dforest.dfuseevs, ref info, dForest.innerobj, rep.innerobj);
-      if (info != 1) throw new ArgumentException("Error in calculation of random forest regression solution");
-
-      rmsError = rep.rmserror;
-      avgRelError = rep.avgrelerror;
-      outOfBagAvgRelError = rep.oobavgrelerror;
-      outOfBagRmsError = rep.oobrmserror;
-
-      return new RandomForestRegressionSolution((IRegressionProblemData)problemData.Clone(), new RandomForestModel(dForest, targetVariable, allowedInputVariables));
+      var model = RandomForestModel.CreateRegressionModel(problemData, nTrees, r, m, seed, out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError);
+      return new RandomForestRegressionSolution((IRegressionProblemData)problemData.Clone(), model);
     }
     #endregion
Index: branches/gteufl/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegressionSolution.cs
===================================================================
--- branches/gteufl/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegressionSolution.cs	(revision 10225)
+++ branches/gteufl/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegressionSolution.cs	(revision 12969)
@@ -1,5 +1,5 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
Index: branches/gteufl/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestUtil.cs
===================================================================
--- branches/gteufl/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestUtil.cs	(revision 12969)
+++ branches/gteufl/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestUtil.cs	(revision 12969)
@@ -0,0 +1,358 @@
+﻿#region License Information
+
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Linq.Expressions;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Random;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("RFParameter", "A random forest parameter collection")]
+  [StorableClass]
+  public class RFParameter : ParameterCollection {
+    public RFParameter() {
+      base.Add(new FixedValueParameter<IntValue>("N", "The number of random forest trees", new IntValue(50)));
+      base.Add(new FixedValueParameter<DoubleValue>("M", "The ratio of features that will be used in the construction of individual trees (0<m<=1)", new DoubleValue(0.1)));
+      base.Add(new FixedValueParameter<DoubleValue>("R", "The ratio of the training set that will be used in the construction of individual trees (0<r<=1)", new DoubleValue(0.1)));
+    }
+
+    [StorableConstructor]
+    protected RFParameter(bool deserializing)
+      : base(deserializing) {
+    }
+
+    protected RFParameter(RFParameter original, Cloner cloner)
+      : base(original, cloner) {
+      this.N = original.N;
+      this.R = original.R;
+      this.M = original.M;
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RFParameter(this, cloner);
+    }
+
+    private IFixedValueParameter<IntValue> NParameter {
+      get { return (IFixedValueParameter<IntValue>)base["N"]; }
+    }
+
+    private IFixedValueParameter<DoubleValue> RParameter {
+      get { return (IFixedValueParameter<DoubleValue>)base["R"]; }
+    }
+
+    private IFixedValueParameter<DoubleValue> MParameter {
+      get { return (IFixedValueParameter<DoubleValue>)base["M"]; }
+    }
+
+    public int N {
+      get { return NParameter.Value.Value; }
+      set { NParameter.Value.Value = value; }
+    }
+
+    public double R {
+      get { return RParameter.Value.Value; }
+      set { RParameter.Value.Value = value; }
+    }
+
+    public double M {
+      get { return MParameter.Value.Value; }
+      set { MParameter.Value.Value = value; }
+    }
+  }
+
+  public static class RandomForestUtil {
+    private static void CrossValidate(IRegressionProblemData problemData, Tuple<IEnumerable<int>, IEnumerable<int>>[] partitions, int nTrees, double r, double m, int seed, out double avgTestMse) {
+      avgTestMse = 0;
+      var ds = problemData.Dataset;
+      var targetVariable = GetTargetVariableName(problemData);
+      foreach (var tuple in partitions) {
+        double rmsError, avgRelError, outOfBagAvgRelError, outOfBagRmsError;
+        var trainingRandomForestPartition = tuple.Item1;
+        var testRandomForestPartition = tuple.Item2;
+        var model = RandomForestModel.CreateRegressionModel(problemData, trainingRandomForestPartition, nTrees, r, m, seed, out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError);
+        var estimatedValues = model.GetEstimatedValues(ds, testRandomForestPartition);
+        var targetValues = ds.GetDoubleValues(targetVariable, testRandomForestPartition);
+        OnlineCalculatorError calculatorError;
+        double mse = OnlineMeanSquaredErrorCalculator.Calculate(estimatedValues, targetValues, out calculatorError);
+        if (calculatorError != OnlineCalculatorError.None)
+          mse = double.NaN;
+        avgTestMse += mse;
+      }
+      avgTestMse /= partitions.Length;
+    }
+
+    private static void CrossValidate(IClassificationProblemData problemData, Tuple<IEnumerable<int>, IEnumerable<int>>[] partitions, int nTrees, double r, double m, int seed, out double avgTestAccuracy) {
+      avgTestAccuracy = 0;
+      var ds = problemData.Dataset;
+      var targetVariable = GetTargetVariableName(problemData);
+      foreach (var tuple in partitions) {
+        double rmsError, avgRelError, outOfBagAvgRelError, outOfBagRmsError;
+        var trainingRandomForestPartition = tuple.Item1;
+        var testRandomForestPartition = tuple.Item2;
+        var model = RandomForestModel.CreateClassificationModel(problemData, trainingRandomForestPartition, nTrees, r, m, seed, out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError);
+        var estimatedValues = model.GetEstimatedClassValues(ds, testRandomForestPartition);
+        var targetValues = ds.GetDoubleValues(targetVariable, testRandomForestPartition);
+        OnlineCalculatorError calculatorError;
+        double accuracy = OnlineAccuracyCalculator.Calculate(estimatedValues, targetValues, out calculatorError);
+        if (calculatorError != OnlineCalculatorError.None)
+          accuracy = double.NaN;
+        avgTestAccuracy += accuracy;
+      }
+      avgTestAccuracy /= partitions.Length;
+    }
+
+    /// <summary>
+    /// Grid search without crossvalidation (since for random forests the out-of-bag estimate is unbiased)
+    /// </summary>
+    /// <param name="problemData">The regression problem data</param>
+    /// <param name="parameterRanges">The ranges for each parameter in the grid search</param>
+    /// <param name="seed">The random seed (required by the random forest model)</param>
+    /// <param name="maxDegreeOfParallelism">The maximum allowed number of threads (to parallelize the grid search)</param>
+    public static RFParameter GridSearch(IRegressionProblemData problemData, Dictionary<string, IEnumerable<double>> parameterRanges, int seed = 12345, int maxDegreeOfParallelism = 1) {
+      var setters = parameterRanges.Keys.Select(GenerateSetter).ToList();
+      var crossProduct = parameterRanges.Values.CartesianProduct();
+      double bestOutOfBagRmsError = double.MaxValue;
+      RFParameter bestParameters = new RFParameter();
+
+      var locker = new object();
+      Parallel.ForEach(crossProduct, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, parameterCombination => {
+        var parameterValues = parameterCombination.ToList();
+        var parameters = new RFParameter();
+        for (int i = 0; i < setters.Count; ++i) { setters[i](parameters, parameterValues[i]); }
+        double rmsError, outOfBagRmsError, avgRelError, outOfBagAvgRelError;
+        RandomForestModel.CreateRegressionModel(problemData, problemData.TrainingIndices, parameters.N, parameters.R, parameters.M, seed, out rmsError, out outOfBagRmsError, out avgRelError, out outOfBagAvgRelError);
+
+        lock (locker) {
+          if (bestOutOfBagRmsError > outOfBagRmsError) {
+            bestOutOfBagRmsError = outOfBagRmsError;
+            bestParameters = (RFParameter)parameters.Clone();
+          }
+        }
+      });
+      return bestParameters;
+    }
+
+    /// <summary>
+    /// Grid search without crossvalidation (since for random forests the out-of-bag estimate is unbiased)
+    /// </summary>
+    /// <param name="problemData">The classification problem data</param>
+    /// <param name="parameterRanges">The ranges for each parameter in the grid search</param>
+    /// <param name="seed">The random seed (required by the random forest model)</param>
+    /// <param name="maxDegreeOfParallelism">The maximum allowed number of threads (to parallelize the grid search)</param>
+    public static RFParameter GridSearch(IClassificationProblemData problemData, Dictionary<string, IEnumerable<double>> parameterRanges, int seed = 12345, int maxDegreeOfParallelism = 1) {
+      var setters = parameterRanges.Keys.Select(GenerateSetter).ToList();
+      var crossProduct = parameterRanges.Values.CartesianProduct();
+
+      double bestOutOfBagRmsError = double.MaxValue;
+      RFParameter bestParameters = new RFParameter();
+
+      var locker = new object();
+      Parallel.ForEach(crossProduct, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, parameterCombination => {
+        var parameterValues = parameterCombination.ToList();
+        var parameters = new RFParameter();
+        for (int i = 0; i < setters.Count; ++i) { setters[i](parameters, parameterValues[i]); }
+        double rmsError, outOfBagRmsError, avgRelError, outOfBagAvgRelError;
+        RandomForestModel.CreateClassificationModel(problemData, problemData.TrainingIndices, parameters.N, parameters.R, parameters.M, seed,
+                                                                out rmsError, out outOfBagRmsError, out avgRelError, out outOfBagAvgRelError);
+
+        lock (locker) {
+          if (bestOutOfBagRmsError > outOfBagRmsError) {
+            bestOutOfBagRmsError = outOfBagRmsError;
+            bestParameters = (RFParameter)parameters.Clone();
+          }
+        }
+      });
+      return bestParameters;
+    }
+
+    /// <summary>
+    /// Grid search with crossvalidation
+    /// </summary>
+    /// <param name="problemData">The regression problem data</param>
+    /// <param name="numberOfFolds">The number of folds for crossvalidation</param>
+    /// <param name="shuffleFolds">Specifies whether the folds should be shuffled</param>
+    /// <param name="parameterRanges">The ranges for each parameter in the grid search</param>
+    /// <param name="seed">The random seed (required by the random forest model)</param>
+    /// <param name="maxDegreeOfParallelism">The maximum allowed number of threads (to parallelize the grid search)</param>
+    /// <returns>The best parameter values found by the grid search</returns>
+    public static RFParameter GridSearch(IRegressionProblemData problemData, int numberOfFolds, bool shuffleFolds, Dictionary<string, IEnumerable<double>> parameterRanges, int seed = 12345, int maxDegreeOfParallelism = 1) {
+      DoubleValue mse = new DoubleValue(Double.MaxValue);
+      RFParameter bestParameter = new RFParameter();
+
+      var setters = parameterRanges.Keys.Select(GenerateSetter).ToList();
+      var partitions = GenerateRandomForestPartitions(problemData, numberOfFolds);
+      var crossProduct = parameterRanges.Values.CartesianProduct();
+
+      var locker = new object();
+      Parallel.ForEach(crossProduct, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, parameterCombination => {
+        var parameterValues = parameterCombination.ToList();
+        double testMSE;
+        var parameters = new RFParameter();
+        for (int i = 0; i < setters.Count; ++i) {
+          setters[i](parameters, parameterValues[i]);
+        }
+        CrossValidate(problemData, partitions, parameters.N, parameters.R, parameters.M, seed, out testMSE);
+
+        lock (locker) {
+          if (testMSE < mse.Value) {
+            mse.Value = testMSE;
+            bestParameter = (RFParameter)parameters.Clone();
+          }
+        }
+      });
+      return bestParameter;
+    }
+
+    /// <summary>
+    /// Grid search with crossvalidation
+    /// </summary>
+    /// <param name="problemData">The classification problem data</param>
+    /// <param name="numberOfFolds">The number of folds for crossvalidation</param>
+    /// <param name="shuffleFolds">Specifies whether the folds should be shuffled</param>
+    /// <param name="parameterRanges">The ranges for each parameter in the grid search</param>
+    /// <param name="seed">The random seed (for shuffling)</param>
+    /// <param name="maxDegreeOfParallelism">The maximum allowed number of threads (to parallelize the grid search)</param>
+    public static RFParameter GridSearch(IClassificationProblemData problemData, int numberOfFolds, bool shuffleFolds, Dictionary<string, IEnumerable<double>> parameterRanges, int seed = 12345, int maxDegreeOfParallelism = 1) {
+      DoubleValue accuracy = new DoubleValue(0);
+      RFParameter bestParameter = new RFParameter();
+
+      var setters = parameterRanges.Keys.Select(GenerateSetter).ToList();
+      var crossProduct = parameterRanges.Values.CartesianProduct();
+      var partitions = GenerateRandomForestPartitions(problemData, numberOfFolds, shuffleFolds);
+
+      var locker = new object();
+      Parallel.ForEach(crossProduct, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, parameterCombination => {
+        var parameterValues = parameterCombination.ToList();
+        double testAccuracy;
+        var parameters = new RFParameter();
+        for (int i = 0; i < setters.Count; ++i) {
+          setters[i](parameters, parameterValues[i]);
+        }
+        CrossValidate(problemData, partitions, parameters.N, parameters.R, parameters.M, seed, out testAccuracy);
+
+        lock (locker) {
+          if (testAccuracy > accuracy.Value) {
+            accuracy.Value = testAccuracy;
+            bestParameter = (RFParameter)parameters.Clone();
+          }
+        }
+      });
+      return bestParameter;
+    }
+
+    private static Tuple<IEnumerable<int>, IEnumerable<int>>[] GenerateRandomForestPartitions(IDataAnalysisProblemData problemData, int numberOfFolds, bool shuffleFolds = false) {
+      var folds = GenerateFolds(problemData, numberOfFolds, shuffleFolds).ToList();
+      var partitions = new Tuple<IEnumerable<int>, IEnumerable<int>>[numberOfFolds];
+
+      for (int i = 0; i < numberOfFolds; ++i) {
+        int p = i; // avoid "access to modified closure" warning
+        var trainingRows = folds.SelectMany((par, j) => j != p ? par : Enumerable.Empty<int>());
+        var testRows = folds[i];
+        partitions[i] = new Tuple<IEnumerable<int>, IEnumerable<int>>(trainingRows, testRows);
+      }
+      return partitions;
+    }
+
+    public static IEnumerable<IEnumerable<int>> GenerateFolds(IDataAnalysisProblemData problemData, int numberOfFolds, bool shuffleFolds = false) {
+      var random = new MersenneTwister((uint)Environment.TickCount);
+      if (problemData is IRegressionProblemData) {
+        var trainingIndices = shuffleFolds ? problemData.TrainingIndices.OrderBy(x => random.Next()) : problemData.TrainingIndices;
+        return GenerateFolds(trainingIndices, problemData.TrainingPartition.Size, numberOfFolds);
+      }
+      if (problemData is IClassificationProblemData) {
+        // when shuffle is enabled do stratified folds generation, some folds may have zero elements 
+        // otherwise, generate folds normally
+        return shuffleFolds ? GenerateFoldsStratified(problemData as IClassificationProblemData, numberOfFolds, random) : GenerateFolds(problemData.TrainingIndices, problemData.TrainingPartition.Size, numberOfFolds);
+      }
+      throw new ArgumentException("Problem data is neither regression or classification problem data.");
+    }
+
+    /// <summary>
+    /// Stratified fold generation from classification data. Stratification means that we ensure the same distribution of class labels for each fold.
+    /// The samples are grouped by class label and each group is split into @numberOfFolds parts. The final folds are formed from the joining of 
+    /// the corresponding parts from each class label.
+    /// </summary>
+    /// <param name="problemData">The classification problem data.</param>
+    /// <param name="numberOfFolds">The number of folds in which to split the data.</param>
+    /// <param name="random">The random generator used to shuffle the folds.</param>
+    /// <returns>An enumerable sequece of folds, where a fold is represented by a sequence of row indices.</returns>
+    private static IEnumerable<IEnumerable<int>> GenerateFoldsStratified(IClassificationProblemData problemData, int numberOfFolds, IRandom random) {
+      var values = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+      var valuesIndices = problemData.TrainingIndices.Zip(values, (i, v) => new { Index = i, Value = v }).ToList();
+      IEnumerable<IEnumerable<IEnumerable<int>>> foldsByClass = valuesIndices.GroupBy(x => x.Value, x => x.Index).Select(g => GenerateFolds(g, g.Count(), numberOfFolds));
+      var enumerators = foldsByClass.Select(f => f.GetEnumerator()).ToList();
+      while (enumerators.All(e => e.MoveNext())) {
+        yield return enumerators.SelectMany(e => e.Current).OrderBy(x => random.Next()).ToList();
+      }
+    }
+
+    private static IEnumerable<IEnumerable<T>> GenerateFolds<T>(IEnumerable<T> values, int valuesCount, int numberOfFolds) {
+      // if number of folds is greater than the number of values, some empty folds will be returned
+      if (valuesCount < numberOfFolds) {
+        for (int i = 0; i < numberOfFolds; ++i)
+          yield return i < valuesCount ? values.Skip(i).Take(1) : Enumerable.Empty<T>();
+      } else {
+        int f = valuesCount / numberOfFolds, r = valuesCount % numberOfFolds; // number of folds rounded to integer and remainder
+        int start = 0, end = f;
+        for (int i = 0; i < numberOfFolds; ++i) {
+          if (r > 0) {
+            ++end;
+            --r;
+          }
+          yield return values.Skip(start).Take(end - start);
+          start = end;
+          end += f;
+        }
+      }
+    }
+
+    private static Action<RFParameter, double> GenerateSetter(string field) {
+      var targetExp = Expression.Parameter(typeof(RFParameter));
+      var valueExp = Expression.Parameter(typeof(double));
+      var fieldExp = Expression.Property(targetExp, field);
+      var assignExp = Expression.Assign(fieldExp, Expression.Convert(valueExp, fieldExp.Type));
+      var setter = Expression.Lambda<Action<RFParameter, double>>(assignExp, targetExp, valueExp).Compile();
+      return setter;
+    }
+
+    private static string GetTargetVariableName(IDataAnalysisProblemData problemData) {
+      var regressionProblemData = problemData as IRegressionProblemData;
+      var classificationProblemData = problemData as IClassificationProblemData;
+
+      if (regressionProblemData != null)
+        return regressionProblemData.TargetVariable;
+      if (classificationProblemData != null)
+        return classificationProblemData.TargetVariable;
+
+      throw new ArgumentException("Problem data is neither regression or classification problem data.");
+    }
+  }
+}
