Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs	(revision 11337)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs	(revision 11338)
@@ -188,10 +188,10 @@
 
     public static RandomForestModel CreateRegressionModel(IRegressionProblemData problemData, int nTrees, double r, double m, int seed,
-      out double rmsError, out double avgRelError, out double outOfBagAvgRelError, out double outOfBagRmsError) {
+      out double rmsError, out double outOfBagRmsError, out double avgRelError, out double outOfBagAvgRelError) {
       return CreateRegressionModel(problemData, nTrees, r, m, seed, out rmsError, out avgRelError, out outOfBagAvgRelError, out outOfBagRmsError, problemData.TrainingIndices);
     }
 
     public static RandomForestModel CreateRegressionModel(IRegressionProblemData problemData, int nTrees, double r, double m, int seed,
-      out double rmsError, out double avgRelError, out double outOfBagAvgRelError, out double outOfBagRmsError, IEnumerable<int> trainingIndices) {
+      out double rmsError, out double outOfBagRmsError, out double avgRelError, out double outOfBagAvgRelError, IEnumerable<int> trainingIndices) {
       var variables = problemData.AllowedInputVariables.Concat(new string[] { problemData.TargetVariable });
       double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(problemData.Dataset, variables, problemData.TrainingIndices);
@@ -212,7 +212,12 @@
     public static RandomForestModel CreateClassificationModel(IClassificationProblemData problemData, int nTrees, double r, double m, int seed,
       out double rmsError, out double outOfBagRmsError, out double relClassificationError, out double outOfBagRelClassificationError) {
+      return CreateClassificationModel(problemData, nTrees, r, m, seed, out rmsError, out outOfBagRmsError, out relClassificationError, out outOfBagRelClassificationError, problemData.TrainingIndices);
+    }
+
+    public static RandomForestModel CreateClassificationModel(IClassificationProblemData problemData, int nTrees, double r, double m, int seed,
+      out double rmsError, out double outOfBagRmsError, out double relClassificationError, out double outOfBagRelClassificationError, IEnumerable<int> trainingIndices) {
 
       var variables = problemData.AllowedInputVariables.Concat(new string[] { problemData.TargetVariable });
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(problemData.Dataset, variables, problemData.TrainingIndices);
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(problemData.Dataset, variables, trainingIndices);
 
       var classValues = problemData.ClassValues.ToArray();
@@ -268,5 +273,5 @@
 
     private static void AssertInputMatrix(double[,] inputMatrix) {
-      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
+      if (inputMatrix.Cast<double>().Any(x => Double.IsNaN(x) || Double.IsInfinity(x)))
         throw new NotSupportedException("Random forest modeling does not support NaN or infinity values in the input dataset.");
     }
Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestUtil.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestUtil.cs	(revision 11337)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestUtil.cs	(revision 11338)
@@ -44,6 +44,4 @@
       var targetExp = Expression.Parameter(typeof(RFParameter));
       var valueExp = Expression.Parameter(typeof(double));
-
-      // Expression.Property can be used here as well
       var fieldExp = Expression.Field(targetExp, field);
       var assignExp = Expression.Assign(fieldExp, Expression.Convert(valueExp, fieldExp.Type));
@@ -53,60 +51,116 @@
 
     /// <summary>
-    /// Generate a collection of training indices corresponding to folds in the data (used for crossvalidation)
+    /// Generate a collection of row sequences corresponding to folds in the data (used for crossvalidation)
     /// </summary>
     /// <remarks>This method is aimed to be lightweight and as such does not clone the dataset.</remarks>
     /// <param name="problemData">The problem data</param>
-    /// <param name="nFolds">The number of folds to generate</param>
+    /// <param name="numberOfFolds">The number of folds to generate</param>
     /// <returns>A sequence of folds representing each a sequence of row numbers</returns>
-    public static IEnumerable<IEnumerable<int>> GenerateFolds(IRegressionProblemData problemData, int nFolds) {
+    public static IEnumerable<IEnumerable<int>> GenerateFolds(IDataAnalysisProblemData problemData, int numberOfFolds) {
       int size = problemData.TrainingPartition.Size;
-
-      int foldSize = size / nFolds; // rounding to integer
-      var trainingIndices = problemData.TrainingIndices;
-
-      for (int i = 0; i < nFolds; ++i) {
-        int n = i * foldSize;
-        int s = n + 2 * foldSize > size ? foldSize + size % foldSize : foldSize;
-        yield return trainingIndices.Skip(n).Take(s);
-      }
-    }
-
-    public static void CrossValidate(IRegressionProblemData problemData, IEnumerable<IEnumerable<int>> folds, RFParameter parameter, int seed, out double avgTestMSE) {
-      CrossValidate(problemData, folds, (int)Math.Round(parameter.n), parameter.m, parameter.r, seed, out avgTestMSE);
-    }
-
-    public static void CrossValidate(IRegressionProblemData problemData, IEnumerable<IEnumerable<int>> folds, int nTrees, double m, double r, int seed, out double avgTestMSE) {
-      avgTestMSE = 0;
-
+      int f = size / numberOfFolds, r = size % numberOfFolds; // number of folds rounded to integer and remainder
+      int start = 0, end = f;
+      for (int i = 0; i < numberOfFolds; ++i) {
+        if (r > 0) { ++end; --r; }
+        yield return problemData.TrainingIndices.Skip(start).Take(end - start);
+        start = end;
+        end += f;
+      }
+    }
+
+    private static Tuple<IEnumerable<int>, IEnumerable<int>>[] GenerateRandomForestPartitions(IDataAnalysisProblemData problemData, int numberOfFolds) {
+      var folds = GenerateFolds(problemData, numberOfFolds).ToList();
+      var partitions = new Tuple<IEnumerable<int>, IEnumerable<int>>[numberOfFolds];
+
+      for (int i = 0; i < numberOfFolds; ++i) {
+        int p = i; // avoid "access to modified closure" warning
+        var trainingRows = folds.SelectMany((par, j) => j != p ? par : Enumerable.Empty<int>());
+        var testRows = folds[i];
+        partitions[i] = new Tuple<IEnumerable<int>, IEnumerable<int>>(trainingRows, testRows);
+      }
+      return partitions;
+    }
+
+    public static void CrossValidate(IDataAnalysisProblemData problemData, int numberOfFolds, RFParameter parameters, int seed, out double error) {
+      var partitions = GenerateRandomForestPartitions(problemData, numberOfFolds);
+      CrossValidate(problemData, partitions, parameters, seed, out error);
+    }
+
+    // user should call the more specific CrossValidate methods
+    public static void CrossValidate(IDataAnalysisProblemData problemData, Tuple<IEnumerable<int>, IEnumerable<int>>[] partitions, RFParameter parameters, int seed, out double error) {
+      CrossValidate(problemData, partitions, (int)parameters.n, parameters.m, parameters.r, seed, out error);
+    }
+
+    public static void CrossValidate(IDataAnalysisProblemData problemData, Tuple<IEnumerable<int>, IEnumerable<int>>[] partitions, int nTrees, double r, double m, int seed, out double error) {
+      var regressionProblemData = problemData as IRegressionProblemData;
+      var classificationProblemData = problemData as IClassificationProblemData;
+      if (regressionProblemData != null)
+        CrossValidate(regressionProblemData, partitions, nTrees, m, r, seed, out error);
+      else if (classificationProblemData != null)
+        CrossValidate(classificationProblemData, partitions, nTrees, m, r, seed, out error);
+      else throw new ArgumentException("Problem data is neither regression or classification problem data.");
+    }
+
+    private static void CrossValidate(IRegressionProblemData problemData, Tuple<IEnumerable<int>, IEnumerable<int>>[] partitions, RFParameter parameters, int seed, out double avgTestMse) {
+      CrossValidate(problemData, partitions, (int)parameters.n, parameters.m, parameters.r, seed, out avgTestMse);
+    }
+
+    private static void CrossValidate(IClassificationProblemData problemData, Tuple<IEnumerable<int>, IEnumerable<int>>[] partitions, RFParameter parameters, int seed, out double avgTestMse) {
+      CrossValidate(problemData, partitions, (int)parameters.n, parameters.m, parameters.r, seed, out avgTestMse);
+    }
+
+    private static void CrossValidate(IRegressionProblemData problemData, Tuple<IEnumerable<int>, IEnumerable<int>>[] partitions, int nTrees, double r, double m, int seed, out double avgTestMse) {
+      avgTestMse = 0;
       var ds = problemData.Dataset;
-      var targetVariable = problemData.TargetVariable;
-
-      var partitions = folds.ToList();
-
-      for (int i = 0; i < partitions.Count; ++i) {
+      var targetVariable = GetTargetVariableName(problemData);
+      foreach (var tuple in partitions) {
         double rmsError, avgRelError, outOfBagAvgRelError, outOfBagRmsError;
-        var test = partitions[i];
-        var training = new List<int>();
-        for (int j = 0; j < i; ++j)
-          training.AddRange(partitions[j]);
-
-        for (int j = i + 1; j < partitions.Count; ++j)
-          training.AddRange(partitions[j]);
-
-        var model = RandomForestModel.CreateRegressionModel(problemData, nTrees, m, r, seed, out rmsError, out avgRelError, out outOfBagAvgRelError, out outOfBagRmsError, training);
-        var estimatedValues = model.GetEstimatedValues(ds, test);
-        var outputValues = ds.GetDoubleValues(targetVariable, test);
-
+        var trainingRandomForestPartition = tuple.Item1;
+        var testRandomForestPartition = tuple.Item2;
+        var model = RandomForestModel.CreateRegressionModel(problemData, nTrees, r, m, seed, out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError, trainingRandomForestPartition);
+        var estimatedValues = model.GetEstimatedValues(ds, testRandomForestPartition);
+        var targetValues = ds.GetDoubleValues(targetVariable, testRandomForestPartition);
         OnlineCalculatorError calculatorError;
-        double mse = OnlineMeanSquaredErrorCalculator.Calculate(estimatedValues, outputValues, out calculatorError);
+        double mse = OnlineMeanSquaredErrorCalculator.Calculate(estimatedValues, targetValues, out calculatorError);
         if (calculatorError != OnlineCalculatorError.None)
           mse = double.NaN;
-        avgTestMSE += mse;
-      }
-
-      avgTestMSE /= partitions.Count;
-    }
-
-    public static RFParameter GridSearch(IRegressionProblemData problemData, IEnumerable<IEnumerable<int>> folds, Dictionary<string, IEnumerable<double>> parameterRanges, int seed = 12345, int maxDegreeOfParallelism = 1) {
+        avgTestMse += mse;
+      }
+      avgTestMse /= partitions.Length;
+    }
+
+    private static void CrossValidate(IClassificationProblemData problemData, Tuple<IEnumerable<int>, IEnumerable<int>>[] partitions, int nTrees, double r, double m, int seed, out double avgTestAccuracy) {
+      avgTestAccuracy = 0;
+      var ds = problemData.Dataset;
+      var targetVariable = GetTargetVariableName(problemData);
+      foreach (var tuple in partitions) {
+        double rmsError, avgRelError, outOfBagAvgRelError, outOfBagRmsError;
+        var trainingRandomForestPartition = tuple.Item1;
+        var testRandomForestPartition = tuple.Item2;
+        var model = RandomForestModel.CreateClassificationModel(problemData, nTrees, r, m, seed, out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError, trainingRandomForestPartition);
+        var estimatedValues = model.GetEstimatedClassValues(ds, testRandomForestPartition);
+        var targetValues = ds.GetDoubleValues(targetVariable, testRandomForestPartition);
+        OnlineCalculatorError calculatorError;
+        double accuracy = OnlineAccuracyCalculator.Calculate(estimatedValues, targetValues, out calculatorError);
+        if (calculatorError != OnlineCalculatorError.None)
+          accuracy = double.NaN;
+        avgTestAccuracy += accuracy;
+      }
+      avgTestAccuracy /= partitions.Length;
+    }
+
+    public static RFParameter GridSearch(IDataAnalysisProblemData problemData, int numberOfFolds, Dictionary<string, IEnumerable<double>> parameterRanges, int seed = 12345, int maxDegreeOfParallelism = 1) {
+      var regressionProblemData = problemData as IRegressionProblemData;
+      var classificationProblemData = problemData as IClassificationProblemData;
+
+      if (regressionProblemData != null)
+        return GridSearch(regressionProblemData, numberOfFolds, parameterRanges, seed, maxDegreeOfParallelism);
+      if (classificationProblemData != null)
+        return GridSearch(classificationProblemData, numberOfFolds, parameterRanges, seed, maxDegreeOfParallelism);
+
+      throw new ArgumentException("Problem data is neither regression or classification problem data.");
+    }
+
+    private static RFParameter GridSearch(IRegressionProblemData problemData, int numberOfFolds, Dictionary<string, IEnumerable<double>> parameterRanges, int seed = 12345, int maxDegreeOfParallelism = 1) {
       DoubleValue mse = new DoubleValue(Double.MaxValue);
       RFParameter bestParameter = new RFParameter { n = 1, m = 0.1, r = 0.1 }; // some random defaults
@@ -115,5 +169,5 @@
       var pRanges = pNames.Select(x => parameterRanges[x]);
       var setters = pNames.Select(GenerateSetter).ToList();
-
+      var partitions = GenerateRandomForestPartitions(problemData, numberOfFolds);
       var crossProduct = pRanges.CartesianProduct();
 
@@ -126,16 +180,51 @@
           s(parameters, list[i]);
         }
-        CrossValidate(problemData, folds, parameters, seed, out testMSE);
+        CrossValidate(problemData, partitions, parameters, seed, out testMSE);
         if (testMSE < mse.Value) {
-          lock (mse) {
-            mse.Value = testMSE;
-          }
-          lock (bestParameter) {
-            bestParameter = (RFParameter)parameters.Clone();
-          }
+          lock (mse) { mse.Value = testMSE; }
+          lock (bestParameter) { bestParameter = (RFParameter)parameters.Clone(); }
         }
       });
       return bestParameter;
     }
+
+    private static RFParameter GridSearch(IClassificationProblemData problemData, int numberOfFolds, Dictionary<string, IEnumerable<double>> parameterRanges, int seed = 12345, int maxDegreeOfParallelism = 1) {
+      DoubleValue accuracy = new DoubleValue(0);
+      RFParameter bestParameter = new RFParameter { n = 1, m = 0.1, r = 0.1 }; // some random defaults
+
+      var pNames = parameterRanges.Keys.ToList();
+      var pRanges = pNames.Select(x => parameterRanges[x]);
+      var setters = pNames.Select(GenerateSetter).ToList();
+      var partitions = GenerateRandomForestPartitions(problemData, numberOfFolds);
+      var crossProduct = pRanges.CartesianProduct();
+
+      Parallel.ForEach(crossProduct, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, nuple => {
+        var list = nuple.ToList();
+        double testAccuracy;
+        var parameters = new RFParameter();
+        for (int i = 0; i < pNames.Count; ++i) {
+          var s = setters[i];
+          s(parameters, list[i]);
+        }
+        CrossValidate(problemData, partitions, parameters, seed, out testAccuracy);
+        if (testAccuracy > accuracy.Value) {
+          lock (accuracy) { accuracy.Value = testAccuracy; }
+          lock (bestParameter) { bestParameter = (RFParameter)parameters.Clone(); }
+        }
+      });
+      return bestParameter;
+    }
+
+    private static string GetTargetVariableName(IDataAnalysisProblemData problemData) {
+      var regressionProblemData = problemData as IRegressionProblemData;
+      var classificationProblemData = problemData as IClassificationProblemData;
+
+      if (regressionProblemData != null)
+        return regressionProblemData.TargetVariable;
+      if (classificationProblemData != null)
+        return classificationProblemData.TargetVariable;
+
+      throw new ArgumentException("Problem data is neither regression or classification problem data.");
+    }
   }
 }
