Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneR.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneR.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneR.cs	(revision 14029)
@@ -0,0 +1,169 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// 1R classification algorithm.
+  /// </summary>
+  [Item("OneR Classification", "A simple classification algorithm the searches the best single-variable split (does not support categorical features correctly). See R.C. Holte (1993). Very simple classification rules perform well on most commonly used datasets. Machine Learning. 11:63-91.")]
+  [StorableClass]
+  public sealed class OneR : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+
+    public IValueParameter<IntValue> MinBucketSizeParameter {
+      get { return (IValueParameter<IntValue>)Parameters["MinBucketSize"]; }
+    }
+
+    [StorableConstructor]
+    private OneR(bool deserializing) : base(deserializing) { }
+
+    private OneR(OneR original, Cloner cloner)
+      : base(original, cloner) { }
+
+    public OneR()
+      : base() {
+      Parameters.Add(new ValueParameter<IntValue>("MinBucketSize", "Minimum size of a bucket for numerical values. (Except for the rightmost bucket)", new IntValue(6)));
+      Problem = new ClassificationProblem();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new OneR(this, cloner);
+    }
+
+    protected override void Run() {
+      var solution = CreateOneRSolution(Problem.ProblemData, MinBucketSizeParameter.Value.Value);
+      Results.Add(new Result("OneR solution", "The 1R classifier.", solution));
+    }
+
+    public static IClassificationSolution CreateOneRSolution(IClassificationProblemData problemData, int minBucketSize = 6) {
+      var bestClassified = 0;
+      List<Split> bestSplits = null;
+      string bestVariable = string.Empty;
+      double bestMissingValuesClass = double.NaN;
+      var classValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+
+      foreach (var variable in problemData.AllowedInputVariables) {
+        var inputValues = problemData.Dataset.GetDoubleValues(variable, problemData.TrainingIndices);
+        var samples = inputValues.Zip(classValues, (i, v) => new Sample(i, v)).OrderBy(s => s.inputValue);
+
+        var missingValuesDistribution = samples.Where(s => double.IsNaN(s.inputValue)).GroupBy(s => s.classValue).ToDictionary(s => s.Key, s => s.Count()).MaxItems(s => s.Value).FirstOrDefault();
+
+        //calculate class distributions for all distinct inputValues
+        List<Dictionary<double, int>> classDistributions = new List<Dictionary<double, int>>();
+        List<double> thresholds = new List<double>();
+        double lastValue = double.NaN;
+        foreach (var sample in samples.Where(s => !double.IsNaN(s.inputValue))) {
+          if (sample.inputValue > lastValue || double.IsNaN(lastValue)) {
+            if (!double.IsNaN(lastValue)) thresholds.Add((lastValue + sample.inputValue) / 2);
+            lastValue = sample.inputValue;
+            classDistributions.Add(new Dictionary<double, int>());
+            foreach (var classValue in problemData.ClassValues)
+              classDistributions[classDistributions.Count - 1][classValue] = 0;
+
+          }
+          classDistributions[classDistributions.Count - 1][sample.classValue]++;
+        }
+        thresholds.Add(double.PositiveInfinity);
+
+        var distribution = classDistributions[0];
+        var threshold = thresholds[0];
+        var splits = new List<Split>();
+
+        for (int i = 1; i < classDistributions.Count; i++) {
+          var samplesInSplit = distribution.Max(d => d.Value);
+          //join splits if there are too few samples in the split or the distributions has the same maximum class value as the current split
+          if (samplesInSplit < minBucketSize ||
+            classDistributions[i].MaxItems(d => d.Value).Select(d => d.Key).Contains(
+              distribution.MaxItems(d => d.Value).Select(d => d.Key).First())) {
+            foreach (var classValue in classDistributions[i])
+              distribution[classValue.Key] += classValue.Value;
+            threshold = thresholds[i];
+          } else {
+            splits.Add(new Split(threshold, distribution.MaxItems(d => d.Value).Select(d => d.Key).First()));
+            distribution = classDistributions[i];
+            threshold = thresholds[i];
+          }
+        }
+        splits.Add(new Split(double.PositiveInfinity, distribution.MaxItems(d => d.Value).Select(d => d.Key).First()));
+
+        int correctClassified = 0;
+        int splitIndex = 0;
+        foreach (var sample in samples.Where(s => !double.IsNaN(s.inputValue))) {
+          while (sample.inputValue >= splits[splitIndex].thresholdValue)
+            splitIndex++;
+          correctClassified += sample.classValue == splits[splitIndex].classValue ? 1 : 0;
+        }
+        correctClassified += missingValuesDistribution.Value;
+
+        if (correctClassified > bestClassified) {
+          bestClassified = correctClassified;
+          bestSplits = splits;
+          bestVariable = variable;
+          bestMissingValuesClass = missingValuesDistribution.Value == 0 ? double.NaN : missingValuesDistribution.Key;
+        }
+      }
+
+      //remove neighboring splits with the same class value
+      for (int i = 0; i < bestSplits.Count - 1; i++) {
+        if (bestSplits[i].classValue == bestSplits[i + 1].classValue) {
+          bestSplits.Remove(bestSplits[i]);
+          i--;
+        }
+      }
+
+      var model = new OneRClassificationModel(problemData.TargetVariable, bestVariable, bestSplits.Select(s => s.thresholdValue).ToArray(), bestSplits.Select(s => s.classValue).ToArray(), bestMissingValuesClass);
+      var solution = new OneRClassificationSolution(model, (IClassificationProblemData)problemData.Clone());
+
+      return solution;
+    }
+
+    #region helper classes
+    private class Split {
+      public double thresholdValue;
+      public double classValue;
+
+      public Split(double thresholdValue, double classValue) {
+        this.thresholdValue = thresholdValue;
+        this.classValue = classValue;
+      }
+    }
+
+    private class Sample {
+      public double inputValue;
+      public double classValue;
+
+      public Sample(double inputValue, double classValue) {
+        this.inputValue = inputValue;
+        this.classValue = classValue;
+      }
+    }
+    #endregion
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationModel.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationModel.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationModel.cs	(revision 14029)
@@ -0,0 +1,119 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item("OneR Classification Model", "A model that uses intervals for one variable to determine the class.")]
+  public class OneRClassificationModel : ClassificationModel {
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return new[] { Variable }; }
+    }
+
+    [Storable]
+    protected string variable;
+    public string Variable {
+      get { return variable; }
+    }
+
+    [Storable]
+    protected double[] splits;
+    public double[] Splits {
+      get { return splits; }
+    }
+
+    [Storable]
+    protected double[] classes;
+    public double[] Classes {
+      get { return classes; }
+    }
+
+    [Storable]
+    protected double missingValuesClass;
+    public double MissingValuesClass {
+      get { return missingValuesClass; }
+    }
+
+    [StorableConstructor]
+    protected OneRClassificationModel(bool deserializing) : base(deserializing) { }
+    protected OneRClassificationModel(OneRClassificationModel original, Cloner cloner)
+      : base(original, cloner) {
+      this.variable = (string)original.variable;
+      this.splits = (double[])original.splits.Clone();
+      this.classes = (double[])original.classes.Clone();
+    }
+    public override IDeepCloneable Clone(Cloner cloner) { return new OneRClassificationModel(this, cloner); }
+
+    public OneRClassificationModel(string targetVariable, string variable, double[] splits, double[] classes, double missingValuesClass = double.NaN)
+      : base(targetVariable) {
+      if (splits.Length != classes.Length) {
+        throw new ArgumentException("Number of splits and classes has to be equal.");
+      }
+      if (!Double.IsPositiveInfinity(splits[splits.Length - 1])) {
+        throw new ArgumentException("Last split has to be double.PositiveInfinity, so that all values are covered.");
+      }
+      this.name = ItemName;
+      this.description = ItemDescription;
+      this.variable = variable;
+      this.splits = splits;
+      this.classes = classes;
+      this.missingValuesClass = missingValuesClass;
+    }
+
+    // uses sorting to return the values in the order of rows, instead of using nested for loops
+    // to avoid O(n²) runtime
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+      var values = dataset.GetDoubleValues(Variable, rows).ToArray();
+      var rowsArray = rows.ToArray();
+      var order = Enumerable.Range(0, rowsArray.Length).ToArray();
+      double[] estimated = new double[rowsArray.Length];
+      Array.Sort(rowsArray, order);
+      Array.Sort(values, rowsArray);
+      int curSplit = 0, curIndex = 0;
+      while (curIndex < values.Length && Double.IsNaN(values[curIndex])) {
+        estimated[curIndex] = MissingValuesClass;
+        curIndex++;
+      }
+      while (curSplit < Splits.Length) {
+        while (curIndex < values.Length && Splits[curSplit] > values[curIndex]) {
+          estimated[curIndex] = classes[curSplit];
+          curIndex++;
+        }
+        curSplit++;
+      }
+      Array.Sort(rowsArray, estimated);
+      Array.Sort(order, estimated);
+      return estimated;
+    }
+
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new OneRClassificationSolution(this, new ClassificationProblemData(problemData));
+    }
+
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationSolution.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationSolution.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationSolution.cs	(revision 14029)
@@ -0,0 +1,48 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "OneR Classification Solution", Description = "Represents a OneR classification solution which uses only a single feature with potentially multiple thresholds for class prediction.")]
+  public class OneRClassificationSolution : ClassificationSolution {
+    public new OneRClassificationModel Model {
+      get { return (OneRClassificationModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    protected OneRClassificationSolution(bool deserializing) : base(deserializing) { }
+    protected OneRClassificationSolution(OneRClassificationSolution original, Cloner cloner) : base(original, cloner) { }
+    public OneRClassificationSolution(OneRClassificationModel model, IClassificationProblemData problemData)
+      : base(model, problemData) {
+      RecalculateResults();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new OneRClassificationSolution(this, cloner);
+    }
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/ZeroR.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/ZeroR.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/ZeroR.cs	(revision 14029)
@@ -0,0 +1,71 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Optimization;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// 0R classification algorithm.
+  /// </summary>
+  [Item("ZeroR Classification", "The simplest possible classifier, ZeroR always predicts the majority class.")]
+  [StorableClass]
+  public sealed class ZeroR : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+
+    [StorableConstructor]
+    private ZeroR(bool deserializing) : base(deserializing) { }
+    private ZeroR(ZeroR original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public ZeroR()
+      : base() {
+      Problem = new ClassificationProblem();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new ZeroR(this, cloner);
+    }
+
+    protected override void Run() {
+      var solution = CreateZeroRSolution(Problem.ProblemData);
+      Results.Add(new Result("ZeroR solution", "The simplest possible classifier, ZeroR always predicts the majority class.", solution));
+    }
+
+    public static IClassificationSolution CreateZeroRSolution(IClassificationProblemData problemData) {
+      var dataset = problemData.Dataset;
+      string target = problemData.TargetVariable;
+      var targetValues = dataset.GetDoubleValues(target, problemData.TrainingIndices);
+
+
+      // if multiple classes have the same number of observations then simply take the first one
+      var dominantClass = targetValues.GroupBy(x => x).ToDictionary(g => g.Key, g => g.Count())
+        .MaxItems(kvp => kvp.Value).Select(x => x.Key).First();
+
+      var model = new ConstantModel(dominantClass, target);
+      var solution = model.CreateClassificationSolution(problemData);
+      return solution;
+    }
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs	(revision 14029)
@@ -35,5 +35,5 @@
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
-  [Item("Cross Validation", "Cross-validation wrapper for data analysis algorithms.")]
+  [Item("Cross Validation (CV)", "Cross-validation wrapper for data analysis algorithms.")]
   [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 100)]
   [StorableClass]
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GBM/GradientBoostingRegressionAlgorithm.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GBM/GradientBoostingRegressionAlgorithm.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GBM/GradientBoostingRegressionAlgorithm.cs	(revision 14029)
@@ -0,0 +1,515 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Analysis;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
+using HeuristicLab.Random;
+using HeuristicLab.Selection;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression {
+  [Item("Gradient Boosting Machine Regression (GBM)",
+    "Gradient boosting for any regression base learner (e.g. MCTS symbolic regression)")]
+  [StorableClass]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 350)]
+  public class GradientBoostingRegressionAlgorithm : BasicAlgorithm {
+    public override Type ProblemType {
+      get { return typeof(IRegressionProblem); }
+    }
+
+    public new IRegressionProblem Problem {
+      get { return (IRegressionProblem)base.Problem; }
+      set { base.Problem = value; }
+    }
+
+    #region ParameterNames
+
+    private const string IterationsParameterName = "Iterations";
+    private const string NuParameterName = "Nu";
+    private const string MParameterName = "M";
+    private const string RParameterName = "R";
+    private const string RegressionAlgorithmParameterName = "RegressionAlgorithm";
+    private const string SeedParameterName = "Seed";
+    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+    private const string CreateSolutionParameterName = "CreateSolution";
+    private const string StoreRunsParameterName = "StoreRuns";
+    private const string RegressionAlgorithmSolutionResultParameterName = "RegressionAlgorithmResult";
+
+    #endregion
+
+    #region ParameterProperties
+
+    public IFixedValueParameter<IntValue> IterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[IterationsParameterName]; }
+    }
+
+    public IFixedValueParameter<DoubleValue> NuParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[NuParameterName]; }
+    }
+
+    public IFixedValueParameter<DoubleValue> RParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[RParameterName]; }
+    }
+
+    public IFixedValueParameter<DoubleValue> MParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[MParameterName]; }
+    }
+
+    // regression algorithms are currently: DataAnalysisAlgorithms, BasicAlgorithms and engine algorithms with no common interface
+    public IConstrainedValueParameter<IAlgorithm> RegressionAlgorithmParameter {
+      get { return (IConstrainedValueParameter<IAlgorithm>)Parameters[RegressionAlgorithmParameterName]; }
+    }
+
+    public IFixedValueParameter<StringValue> RegressionAlgorithmSolutionResultParameter {
+      get { return (IFixedValueParameter<StringValue>)Parameters[RegressionAlgorithmSolutionResultParameterName]; }
+    }
+
+    public IFixedValueParameter<IntValue> SeedParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+
+    public FixedValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (FixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+
+    public IFixedValueParameter<BoolValue> CreateSolutionParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> StoreRunsParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[StoreRunsParameterName]; }
+    }
+
+    #endregion
+
+    #region Properties
+
+    public int Iterations {
+      get { return IterationsParameter.Value.Value; }
+      set { IterationsParameter.Value.Value = value; }
+    }
+
+    public int Seed {
+      get { return SeedParameter.Value.Value; }
+      set { SeedParameter.Value.Value = value; }
+    }
+
+    public bool SetSeedRandomly {
+      get { return SetSeedRandomlyParameter.Value.Value; }
+      set { SetSeedRandomlyParameter.Value.Value = value; }
+    }
+
+    public double Nu {
+      get { return NuParameter.Value.Value; }
+      set { NuParameter.Value.Value = value; }
+    }
+
+    public double R {
+      get { return RParameter.Value.Value; }
+      set { RParameter.Value.Value = value; }
+    }
+
+    public double M {
+      get { return MParameter.Value.Value; }
+      set { MParameter.Value.Value = value; }
+    }
+
+    public bool CreateSolution {
+      get { return CreateSolutionParameter.Value.Value; }
+      set { CreateSolutionParameter.Value.Value = value; }
+    }
+
+    public bool StoreRuns {
+      get { return StoreRunsParameter.Value.Value; }
+      set { StoreRunsParameter.Value.Value = value; }
+    }
+
+    public IAlgorithm RegressionAlgorithm {
+      get { return RegressionAlgorithmParameter.Value; }
+    }
+
+    public string RegressionAlgorithmResult {
+      get { return RegressionAlgorithmSolutionResultParameter.Value.Value; }
+      set { RegressionAlgorithmSolutionResultParameter.Value.Value = value; }
+    }
+
+    #endregion
+
+    [StorableConstructor]
+    protected GradientBoostingRegressionAlgorithm(bool deserializing)
+      : base(deserializing) {
+    }
+
+    protected GradientBoostingRegressionAlgorithm(GradientBoostingRegressionAlgorithm original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GradientBoostingRegressionAlgorithm(this, cloner);
+    }
+
+    public GradientBoostingRegressionAlgorithm() {
+      Problem = new RegressionProblem(); // default problem
+      var osgp = CreateOSGP();
+      var regressionAlgs = new ItemSet<IAlgorithm>(new IAlgorithm[] {
+        new RandomForestRegression(),
+        osgp,
+      });
+      foreach (var alg in regressionAlgs) alg.Prepare();
+
+
+      Parameters.Add(new FixedValueParameter<IntValue>(IterationsParameterName,
+        "Number of iterations", new IntValue(100)));
+      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName,
+        "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName,
+        "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(NuParameterName,
+        "The learning rate nu when updating predictions in GBM (0 < nu <= 1)", new DoubleValue(0.5)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(RParameterName,
+        "The fraction of rows that are sampled randomly for the base learner in each iteration (0 < r <= 1)",
+        new DoubleValue(1)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName,
+        "The fraction of variables that are sampled randomly for the base learner in each iteration (0 < m <= 1)",
+        new DoubleValue(0.5)));
+      Parameters.Add(new ConstrainedValueParameter<IAlgorithm>(RegressionAlgorithmParameterName,
+        "The regression algorithm to use as a base learner", regressionAlgs, osgp));
+      Parameters.Add(new FixedValueParameter<StringValue>(RegressionAlgorithmSolutionResultParameterName,
+        "The name of the solution produced by the regression algorithm", new StringValue("Solution")));
+      Parameters[RegressionAlgorithmSolutionResultParameterName].Hidden = true;
+      Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName,
+        "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+      Parameters[CreateSolutionParameterName].Hidden = true;
+      Parameters.Add(new FixedValueParameter<BoolValue>(StoreRunsParameterName,
+        "Flag that indicates if the results of the individual runs should be stored for detailed analysis", new BoolValue(false)));
+      Parameters[StoreRunsParameterName].Hidden = true;
+    }
+
+    protected override void Run(CancellationToken cancellationToken) {
+      // Set up the algorithm
+      if (SetSeedRandomly) Seed = new System.Random().Next();
+      var rand = new MersenneTwister((uint)Seed);
+
+      // Set up the results display
+      var iterations = new IntValue(0);
+      Results.Add(new Result("Iterations", iterations));
+
+      var table = new DataTable("Qualities");
+      table.Rows.Add(new DataRow("R² (train)"));
+      table.Rows.Add(new DataRow("R² (test)"));
+      Results.Add(new Result("Qualities", table));
+      var curLoss = new DoubleValue();
+      var curTestLoss = new DoubleValue();
+      Results.Add(new Result("R² (train)", curLoss));
+      Results.Add(new Result("R² (test)", curTestLoss));
+      var runCollection = new RunCollection();
+      if (StoreRuns)
+        Results.Add(new Result("Runs", runCollection));
+
+      // init
+      var problemData = Problem.ProblemData;
+      var targetVarName = problemData.TargetVariable;
+      var activeVariables = problemData.AllowedInputVariables.Concat(new string[] { problemData.TargetVariable });
+      var modifiableDataset = new ModifiableDataset(
+        activeVariables,
+        activeVariables.Select(v => problemData.Dataset.GetDoubleValues(v).ToList()));
+
+      var trainingRows = problemData.TrainingIndices;
+      var testRows = problemData.TestIndices;
+      var yPred = new double[trainingRows.Count()];
+      var yPredTest = new double[testRows.Count()];
+      var y = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices).ToArray();
+      var curY = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices).ToArray();
+
+      var yTest = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TestIndices).ToArray();
+      var curYTest = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TestIndices).ToArray();
+      var nu = Nu;
+      var mVars = (int)Math.Ceiling(M * problemData.AllowedInputVariables.Count());
+      var rRows = (int)Math.Ceiling(R * problemData.TrainingIndices.Count());
+      var alg = RegressionAlgorithm;
+      List<IRegressionModel> models = new List<IRegressionModel>();
+      try {
+
+        // Loop until iteration limit reached or canceled.
+        for (int i = 0; i < Iterations; i++) {
+          cancellationToken.ThrowIfCancellationRequested();
+
+          modifiableDataset.RemoveVariable(targetVarName);
+          modifiableDataset.AddVariable(targetVarName, curY.Concat(curYTest));
+
+          SampleTrainingData(rand, modifiableDataset, rRows, problemData.Dataset, curY, problemData.TargetVariable, problemData.TrainingIndices); // all training indices from the original problem data are allowed 
+          var modifiableProblemData = new RegressionProblemData(modifiableDataset,
+            problemData.AllowedInputVariables.SampleRandomWithoutRepetition(rand, mVars),
+            problemData.TargetVariable);
+          modifiableProblemData.TrainingPartition.Start = 0;
+          modifiableProblemData.TrainingPartition.End = rRows;
+          modifiableProblemData.TestPartition.Start = problemData.TestPartition.Start;
+          modifiableProblemData.TestPartition.End = problemData.TestPartition.End;
+
+          if (!TrySetProblemData(alg, modifiableProblemData))
+            throw new NotSupportedException("The algorithm cannot be used with GBM.");
+
+          IRegressionModel model;
+          IRun run;
+
+          // try to find a model. The algorithm might fail to produce a model. In this case we just retry until the iterations are exhausted
+          if (TryExecute(alg, rand.Next(), RegressionAlgorithmResult, out model, out run)) {
+            int row = 0;
+            // update predictions for training and test
+            // update new targets (in the case of squared error loss we simply use negative residuals)
+            foreach (var pred in model.GetEstimatedValues(problemData.Dataset, trainingRows)) {
+              yPred[row] = yPred[row] + nu * pred;
+              curY[row] = y[row] - yPred[row];
+              row++;
+            }
+            row = 0;
+            foreach (var pred in model.GetEstimatedValues(problemData.Dataset, testRows)) {
+              yPredTest[row] = yPredTest[row] + nu * pred;
+              curYTest[row] = yTest[row] - yPredTest[row];
+              row++;
+            }
+            // determine quality
+            OnlineCalculatorError error;
+            var trainR = OnlinePearsonsRCalculator.Calculate(yPred, y, out error);
+            var testR = OnlinePearsonsRCalculator.Calculate(yPredTest, yTest, out error);
+
+            // iteration results
+            curLoss.Value = error == OnlineCalculatorError.None ? trainR * trainR : 0.0;
+            curTestLoss.Value = error == OnlineCalculatorError.None ? testR * testR : 0.0;
+
+            models.Add(model);
+
+
+          }
+
+          if (StoreRuns)
+            runCollection.Add(run);
+          table.Rows["R² (train)"].Values.Add(curLoss.Value);
+          table.Rows["R² (test)"].Values.Add(curTestLoss.Value);
+          iterations.Value = i + 1;
+        }
+
+        // produce solution 
+        if (CreateSolution) {
+          // when all our models are symbolic models we can easily combine them to a single model
+          if (models.All(m => m is ISymbolicRegressionModel)) {
+            Results.Add(new Result("Solution", CreateSymbolicSolution(models, Nu, (IRegressionProblemData)problemData.Clone())));
+          }
+          // just produce an ensemble solution for now (TODO: correct scaling or linear regression for ensemble model weights)
+
+          var ensembleSolution = CreateEnsembleSolution(models, (IRegressionProblemData)problemData.Clone());
+          Results.Add(new Result("EnsembleSolution", ensembleSolution));
+        }
+      }
+      finally {
+        // reset everything
+        alg.Prepare(true);
+      }
+    }
+
+    private static IRegressionEnsembleSolution CreateEnsembleSolution(List<IRegressionModel> models,
+      IRegressionProblemData problemData) {
+      var rows = problemData.TrainingPartition.Size;
+      var features = models.Count;
+      double[,] inputMatrix = new double[rows, features + 1];
+      //add model estimates
+      for (int m = 0; m < models.Count; m++) {
+        var model = models[m];
+        var estimates = model.GetEstimatedValues(problemData.Dataset, problemData.TrainingIndices);
+        int estimatesCounter = 0;
+        foreach (var estimate in estimates) {
+          inputMatrix[estimatesCounter, m] = estimate;
+          estimatesCounter++;
+        }
+      }
+
+      //add target
+      var targets = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+      int targetCounter = 0;
+      foreach (var target in targets) {
+        inputMatrix[targetCounter, models.Count] = target;
+        targetCounter++;
+      }
+
+      alglib.linearmodel lm = new alglib.linearmodel();
+      alglib.lrreport ar = new alglib.lrreport();
+      double[] coefficients;
+      int retVal = 1;
+      alglib.lrbuildz(inputMatrix, rows, features, out retVal, out lm, out ar);
+      if (retVal != 1) throw new ArgumentException("Error in calculation of linear regression solution");
+
+      alglib.lrunpack(lm, out coefficients, out features);
+
+      var ensembleModel = new RegressionEnsembleModel(models, coefficients.Take(models.Count)) { AverageModelEstimates = false };
+      var ensembleSolution = (IRegressionEnsembleSolution)ensembleModel.CreateRegressionSolution(problemData);
+      return ensembleSolution;
+    }
+
+
+    private IAlgorithm CreateOSGP() {
+      // configure strict osgp
+      var alg = new OffspringSelectionGeneticAlgorithm.OffspringSelectionGeneticAlgorithm();
+      var prob = new SymbolicRegressionSingleObjectiveProblem();
+      prob.MaximumSymbolicExpressionTreeDepth.Value = 7;
+      prob.MaximumSymbolicExpressionTreeLength.Value = 15;
+      alg.Problem = prob;
+      alg.SuccessRatio.Value = 1.0;
+      alg.ComparisonFactorLowerBound.Value = 1.0;
+      alg.ComparisonFactorUpperBound.Value = 1.0;
+      alg.MutationProbability.Value = 0.15;
+      alg.PopulationSize.Value = 200;
+      alg.MaximumSelectionPressure.Value = 100;
+      alg.MaximumEvaluatedSolutions.Value = 20000;
+      alg.SelectorParameter.Value = alg.SelectorParameter.ValidValues.OfType<GenderSpecificSelector>().First();
+      alg.MutatorParameter.Value = alg.MutatorParameter.ValidValues.OfType<MultiSymbolicExpressionTreeManipulator>().First();
+      alg.StoreAlgorithmInEachRun = false;
+      return alg;
+    }
+
+    private void SampleTrainingData(MersenneTwister rand, ModifiableDataset ds, int rRows,
+      IDataset sourceDs, double[] curTarget, string targetVarName, IEnumerable<int> trainingIndices) {
+      var selectedRows = trainingIndices.SampleRandomWithoutRepetition(rand, rRows).ToArray();
+      int t = 0;
+      object[] srcRow = new object[ds.Columns];
+      var varNames = ds.DoubleVariables.ToArray();
+      foreach (var r in selectedRows) {
+        // take all values from the original dataset
+        for (int c = 0; c < srcRow.Length; c++) {
+          var col = sourceDs.GetReadOnlyDoubleValues(varNames[c]);
+          srcRow[c] = col[r];
+        }
+        ds.ReplaceRow(t, srcRow);
+        // but use the updated target values
+        ds.SetVariableValue(curTarget[r], targetVarName, t);
+        t++;
+      }
+    }
+
+    private static ISymbolicRegressionSolution CreateSymbolicSolution(List<IRegressionModel> models, double nu, IRegressionProblemData problemData) {
+      var symbModels = models.OfType<ISymbolicRegressionModel>();
+      var lowerLimit = symbModels.Min(m => m.LowerEstimationLimit);
+      var upperLimit = symbModels.Max(m => m.UpperEstimationLimit);
+      var interpreter = new SymbolicDataAnalysisExpressionTreeLinearInterpreter();
+      var progRootNode = new ProgramRootSymbol().CreateTreeNode();
+      var startNode = new StartSymbol().CreateTreeNode();
+
+      var addNode = new Addition().CreateTreeNode();
+      var mulNode = new Multiplication().CreateTreeNode();
+      var scaleNode = (ConstantTreeNode)new Constant().CreateTreeNode(); // all models are scaled using the same nu
+      scaleNode.Value = nu;
+
+      foreach (var m in symbModels) {
+        var relevantPart = m.SymbolicExpressionTree.Root.GetSubtree(0).GetSubtree(0); // skip root and start
+        addNode.AddSubtree((ISymbolicExpressionTreeNode)relevantPart.Clone());
+      }
+
+      mulNode.AddSubtree(addNode);
+      mulNode.AddSubtree(scaleNode);
+      startNode.AddSubtree(mulNode);
+      progRootNode.AddSubtree(startNode);
+      var t = new SymbolicExpressionTree(progRootNode);
+      var combinedModel = new SymbolicRegressionModel(problemData.TargetVariable, t, interpreter, lowerLimit, upperLimit);
+      var sol = new SymbolicRegressionSolution(combinedModel, problemData);
+      return sol;
+    }
+
+    private static bool TrySetProblemData(IAlgorithm alg, IRegressionProblemData problemData) {
+      var prob = alg.Problem as IRegressionProblem;
+      // there is already a problem and it is compatible -> just set problem data
+      if (prob != null) {
+        prob.ProblemDataParameter.Value = problemData;
+        return true;
+      } else return false;
+    }
+
+    private static bool TryExecute(IAlgorithm alg, int seed, string regressionAlgorithmResultName, out IRegressionModel model, out IRun run) {
+      model = null;
+      SetSeed(alg, seed);
+      using (var wh = new AutoResetEvent(false)) {
+        Exception ex = null;
+        EventHandler<EventArgs<Exception>> handler = (sender, args) => {
+          ex = args.Value;
+          wh.Set();
+        };
+        EventHandler handler2 = (sender, args) => wh.Set();
+        alg.ExceptionOccurred += handler;
+        alg.Stopped += handler2;
+        try {
+          alg.Prepare();
+          alg.Start();
+          wh.WaitOne();
+
+          if (ex != null) throw new AggregateException(ex);
+          run = alg.Runs.Last();
+          alg.Runs.Clear();
+          var sols = alg.Results.Select(r => r.Value).OfType<IRegressionSolution>();
+          if (!sols.Any()) return false;
+          var sol = sols.First();
+          if (sols.Skip(1).Any()) {
+            // more than one solution => use regressionAlgorithmResult
+            if (alg.Results.ContainsKey(regressionAlgorithmResultName)) {
+              sol = (IRegressionSolution)alg.Results[regressionAlgorithmResultName].Value;
+            }
+          }
+          var symbRegSol = sol as SymbolicRegressionSolution;
+          // only accept symb reg solutions that do not hit the estimation limits
+          // NaN evaluations would not be critical but are problematic if we want to combine all symbolic models into a single symbolic model 
+          if (symbRegSol == null ||
+            (symbRegSol.TrainingLowerEstimationLimitHits == 0 && symbRegSol.TrainingUpperEstimationLimitHits == 0 &&
+             symbRegSol.TestLowerEstimationLimitHits == 0 && symbRegSol.TestUpperEstimationLimitHits == 0) &&
+            symbRegSol.TrainingNaNEvaluations == 0 && symbRegSol.TestNaNEvaluations == 0) {
+            model = sol.Model;
+          }
+        }
+        finally {
+          alg.ExceptionOccurred -= handler;
+          alg.Stopped -= handler2;
+        }
+      }
+      return model != null;
+    }
+
+    private static void SetSeed(IAlgorithm alg, int seed) {
+      // no common interface for algs that use a PRNG -> use naming convention to set seed
+      var paramItem = alg as IParameterizedItem;
+
+      if (paramItem.Parameters.ContainsKey("SetSeedRandomly")) {
+        ((BoolValue)paramItem.Parameters["SetSeedRandomly"].ActualValue).Value = false;
+        ((IntValue)paramItem.Parameters["Seed"].ActualValue).Value = seed;
+      } else {
+        throw new ArgumentException("Base learner does not have a seed parameter (algorithm {0})", alg.Name);
+      }
+
+    }
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceConst.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceConst.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceConst.cs	(revision 14029)
@@ -21,6 +21,4 @@
 
 using System;
-using System.Collections.Generic;
-using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -83,5 +81,5 @@
     }
 
-    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
       double scale;
       GetParameterValues(p, out scale);
@@ -91,14 +89,10 @@
       cov.CrossCovariance = (x, xt, i, j) => scale;
       if (HasFixedScaleParameter) {
-        cov.CovarianceGradient = (x, i, j) => Enumerable.Empty<double>();
+        cov.CovarianceGradient = (x, i, j) => new double[0];
       } else {
-        cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, scale, columnIndices);
+        cov.CovarianceGradient = (x, i, j) => new[] { 2.0 * scale };
       }
       return cov;
     }
-
-    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, double scale, IEnumerable<int> columnIndices) {
-      yield return 2.0 * scale;
-    }
   }
 }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinear.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinear.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinear.cs	(revision 14029)
@@ -21,6 +21,4 @@
 
 using System;
-using System.Collections.Generic;
-using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -52,11 +50,11 @@
     }
 
-    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
       if (p.Length > 0) throw new ArgumentException("No parameters are allowed for the linear covariance function.");
       // create functions
       var cov = new ParameterizedCovarianceFunction();
-      cov.Covariance = (x, i, j) => Util.ScalarProd(x, i, j, 1, columnIndices);
-      cov.CrossCovariance = (x, xt, i, j) =>  Util.ScalarProd(x, i, xt, j, 1.0 , columnIndices);
-      cov.CovarianceGradient = (x, i, j) => Enumerable.Empty<double>();
+      cov.Covariance = (x, i, j) => Util.ScalarProd(x, i, j, columnIndices, 1.0);
+      cov.CrossCovariance = (x, xt, i, j) => Util.ScalarProd(x, i, xt, j, columnIndices, 1.0);
+      cov.CovarianceGradient = (x, i, j) => new double[0];
       return cov;
     }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinearArd.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinearArd.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinearArd.cs	(revision 14029)
@@ -81,5 +81,5 @@
     }
 
-    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
       double[] inverseLength;
       GetParameterValues(p, out inverseLength);
@@ -90,5 +90,5 @@
       cov.CrossCovariance = (x, xt, i, j) => Util.ScalarProd(x, i, xt, j, inverseLength, columnIndices);
       if (fixedInverseLength)
-        cov.CovarianceGradient = (x, i, j) => Enumerable.Empty<double>();
+        cov.CovarianceGradient = (x, i, j) => new double[0];
       else
         cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, inverseLength, columnIndices);
@@ -96,10 +96,13 @@
     }
 
-    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, double[] inverseLength, IEnumerable<int> columnIndices) {
+    private static IList<double> GetGradient(double[,] x, int i, int j, double[] inverseLength, int[] columnIndices) {
       int k = 0;
-      foreach (int columnIndex in columnIndices) {
-        yield return -2.0 * x[i, columnIndex] * x[j, columnIndex] * inverseLength[k] * inverseLength[k];
+      var g = new List<double>(columnIndices.Length);
+      for (int c = 0; c < columnIndices.Length; c++) {
+        var columnIndex = columnIndices[c];
+        g.Add(-2.0 * x[i, columnIndex] * x[j, columnIndex] * inverseLength[k] * inverseLength[k]);
         k++;
       }
+      return g;
     }
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMask.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMask.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMask.cs	(revision 14029)
@@ -20,8 +20,5 @@
 #endregion
 
-using System;
-using System.Collections.Generic;
 using System.Linq;
-using System.Linq.Expressions;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -73,9 +70,9 @@
     }
 
-    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
       var cov = CovarianceFunctionParameter.Value;
       var selectedDimensions = SelectedDimensionsParameter.Value;
 
-      return cov.GetParameterizedCovarianceFunction(p, selectedDimensions.Intersect(columnIndices));
+      return cov.GetParameterizedCovarianceFunction(p, selectedDimensions.Intersect(columnIndices).ToArray());
     }
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMaternIso.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMaternIso.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMaternIso.cs	(revision 14029)
@@ -111,5 +111,5 @@
     }
 
-    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
       double inverseLength, scale;
       int d = DParameter.Value.Value;
@@ -122,9 +122,9 @@
         double dist = i == j
                        ? 0.0
-                       : Math.Sqrt(Util.SqrDist(x, i, j, Math.Sqrt(d) * inverseLength, columnIndices));
+                       : Math.Sqrt(Util.SqrDist(x, i, j, columnIndices, Math.Sqrt(d) * inverseLength));
         return scale * m(d, dist);
       };
       cov.CrossCovariance = (x, xt, i, j) => {
-        double dist = Math.Sqrt(Util.SqrDist(x, i, xt, j, Math.Sqrt(d) * inverseLength, columnIndices));
+        double dist = Math.Sqrt(Util.SqrDist(x, i, xt, j, columnIndices, Math.Sqrt(d) * inverseLength));
         return scale * m(d, dist);
       };
@@ -155,13 +155,14 @@
     }
 
-
-    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, int d, double scale, double inverseLength, IEnumerable<int> columnIndices,
+    private static IList<double> GetGradient(double[,] x, int i, int j, int d, double scale, double inverseLength, int[] columnIndices,
       bool fixedInverseLength, bool fixedScale) {
       double dist = i == j
                    ? 0.0
-                   : Math.Sqrt(Util.SqrDist(x, i, j, Math.Sqrt(d) * inverseLength, columnIndices));
+                   : Math.Sqrt(Util.SqrDist(x, i, j, columnIndices, Math.Sqrt(d) * inverseLength));
 
-      if (!fixedInverseLength) yield return scale * dm(d, dist);
-      if (!fixedScale) yield return 2 * scale * m(d, dist);
+      var g = new List<double>(2);
+      if (!fixedInverseLength) g.Add(scale * dm(d, dist));
+      if (!fixedScale) g.Add(2 * scale * m(d, dist));
+      return g;
     }
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNeuralNetwork.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNeuralNetwork.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNeuralNetwork.cs	(revision 14029)
@@ -102,5 +102,5 @@
     }
 
-    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
       double length, scale;
       GetParameterValues(p, out scale, out length);
@@ -113,5 +113,6 @@
         double s1 = 1.0;
         double s2 = 1.0;
-        foreach (var col in columnIndices) {
+        for (int c = 0; c < columnIndices.Length; c++) {
+          var col = columnIndices[c];
           sx += x[i, col] * x[j, col];
           s1 += x[i, col] * x[i, col];
@@ -125,5 +126,6 @@
         double s1 = 1.0;
         double s2 = 1.0;
-        foreach (var col in columnIndices) {
+        for (int c = 0; c < columnIndices.Length; c++) {
+          var col = columnIndices[c];
           sx += x[i, col] * xt[j, col];
           s1 += x[i, col] * x[i, col];
@@ -138,24 +140,22 @@
 
     // order of returned gradients must match the order in GetParameterValues!
-    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, double length, double scale, IEnumerable<int> columnIndices,
+    private static IList<double> GetGradient(double[,] x, int i, int j, double length, double scale, int[] columnIndices,
       bool fixedLength, bool fixedScale) {
-      {
-        double sx = 1.0;
-        double s1 = 1.0;
-        double s2 = 1.0;
-        foreach (var col in columnIndices) {
-          sx += x[i, col] * x[j, col];
-          s1 += x[i, col] * x[i, col];
-          s2 += x[j, col] * x[j, col];
-        }
-        var h = (length + s1) * (length + s2);
-        var f = sx / Math.Sqrt(h);
-        if (!fixedLength) {
-          yield return -scale / Math.Sqrt(1.0 - f * f) * ((length * sx * (2.0 * length + s1 + s2)) / Math.Pow(h, 3.0 / 2.0));
-        }
-        if (!fixedScale) {
-          yield return 2.0 * scale * Math.Asin(f);
-        }
+      double sx = 1.0;
+      double s1 = 1.0;
+      double s2 = 1.0;
+      for (int c = 0; c < columnIndices.Length; c++) {
+        var col = columnIndices[c];
+        sx += x[i, col] * x[j, col];
+        s1 += x[i, col] * x[i, col];
+        s2 += x[j, col] * x[j, col];
       }
+      var h = (length + s1) * (length + s2);
+      var f = sx / Math.Sqrt(h);
+
+      var g = new List<double>(2);
+      if (!fixedLength) g.Add(-scale / Math.Sqrt(1.0 - f * f) * ((length * sx * (2.0 * length + s1 + s2)) / Math.Pow(h, 3.0 / 2.0)));
+      if (!fixedScale) g.Add(2.0 * scale * Math.Asin(f));
+      return g;
     }
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNoise.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNoise.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNoise.cs	(revision 14029)
@@ -21,6 +21,4 @@
 
 using System;
-using System.Collections.Generic;
-using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -84,5 +82,5 @@
     }
 
-    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
       double scale;
       GetParameterValues(p, out scale);
@@ -91,9 +89,9 @@
       var cov = new ParameterizedCovarianceFunction();
       cov.Covariance = (x, i, j) => i == j ? scale : 0.0;
-      cov.CrossCovariance = (x, xt, i, j) => Util.SqrDist(x, i, xt, j, 1.0, columnIndices) < 1e-9 ? scale : 0.0;
+      cov.CrossCovariance = (x, xt, i, j) => Util.SqrDist(x, i, xt, j, columnIndices, 1.0) < 1e-9 ? scale : 0.0;
       if (fixedScale)
-        cov.CovarianceGradient = (x, i, j) => Enumerable.Empty<double>();
+        cov.CovarianceGradient = (x, i, j) => new double[0];
       else
-        cov.CovarianceGradient = (x, i, j) => Enumerable.Repeat(i == j ? 2.0 * scale : 0.0, 1);
+        cov.CovarianceGradient = (x, i, j) => new double[1] { i == j ? 2.0 * scale : 0.0 };
       return cov;
     }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePeriodic.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePeriodic.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePeriodic.cs	(revision 14029)
@@ -22,5 +22,4 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -117,5 +116,5 @@
     }
 
-    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
       double inverseLength, period, scale;
       GetParameterValues(p, out scale, out period, out inverseLength);
@@ -145,22 +144,23 @@
     }
 
-
-    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, IEnumerable<int> columnIndices, double scale, double period, double inverseLength,
+    private static IList<double> GetGradient(double[,] x, int i, int j, int[] columnIndices, double scale, double period, double inverseLength,
       bool fixedInverseLength, bool fixedPeriod, bool fixedScale) {
       double k = i == j ? 0.0 : Math.PI * GetDistance(x, x, i, j, columnIndices) / period;
       double gradient = Math.Sin(k) * inverseLength;
       gradient *= gradient;
-      if (!fixedInverseLength) yield return 4.0 * scale * Math.Exp(-2.0 * gradient) * gradient;
+      var g = new List<double>(3);
+      if (!fixedInverseLength) 
+        g.Add(4.0 * scale * Math.Exp(-2.0 * gradient) * gradient);
       if (!fixedPeriod) {
         double r = Math.Sin(k) * inverseLength;
-        yield return 2.0 * k * scale * Math.Exp(-2 * r * r) * Math.Sin(2 * k) * inverseLength * inverseLength;
+        g.Add(2.0 * k * scale * Math.Exp(-2 * r * r) * Math.Sin(2 * k) * inverseLength * inverseLength);
       }
       if (!fixedScale)
-        yield return 2.0 * scale * Math.Exp(-2 * gradient);
-
+        g.Add(2.0 * scale * Math.Exp(-2 * gradient));
+      return g;
     }
 
-    private static double GetDistance(double[,] x, double[,] xt, int i, int j, IEnumerable<int> columnIndices) {
-      return Math.Sqrt(Util.SqrDist(x, i, xt, j, 1, columnIndices));
+    private static double GetDistance(double[,] x, double[,] xt, int i, int j, int[] columnIndices) {
+      return Math.Sqrt(Util.SqrDist(x, i, xt, j, columnIndices, 1));
     }
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePiecewisePolynomial.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePiecewisePolynomial.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePiecewisePolynomial.cs	(revision 14029)
@@ -69,8 +69,8 @@
       Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale parameter of the piecewise polynomial covariance function."));
 
-      var validValues = new ItemSet<IntValue>(new IntValue[] { 
-        (IntValue)(new IntValue().AsReadOnly()), 
-        (IntValue)(new IntValue(1).AsReadOnly()), 
-        (IntValue)(new IntValue(2).AsReadOnly()), 
+      var validValues = new ItemSet<IntValue>(new IntValue[] {
+        (IntValue)(new IntValue().AsReadOnly()),
+        (IntValue)(new IntValue(1).AsReadOnly()),
+        (IntValue)(new IntValue(2).AsReadOnly()),
         (IntValue)(new IntValue(3).AsReadOnly()) });
       Parameters.Add(new ConstrainedValueParameter<IntValue>("V", "The v parameter of the piecewise polynomial function (allowed values 0, 1, 2, 3).", validValues, validValues.First()));
@@ -113,5 +113,5 @@
     }
 
-    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
       double length, scale;
       int v = VParameter.Value.Value;
@@ -148,9 +148,9 @@
       var cov = new ParameterizedCovarianceFunction();
       cov.Covariance = (x, i, j) => {
-        double k = Math.Sqrt(Util.SqrDist(x, i, x, j, 1.0 / length, columnIndices));
+        double k = Math.Sqrt(Util.SqrDist(x, i, x, j, columnIndices, 1.0 / length));
         return scale * Math.Pow(Math.Max(1 - k, 0), exp + v) * f(k);
       };
       cov.CrossCovariance = (x, xt, i, j) => {
-        double k = Math.Sqrt(Util.SqrDist(x, i, xt, j, 1.0 / length, columnIndices));
+        double k = Math.Sqrt(Util.SqrDist(x, i, xt, j, columnIndices, 1.0 / length));
         return scale * Math.Pow(Math.Max(1 - k, 0), exp + v) * f(k);
       };
@@ -159,9 +159,11 @@
     }
 
-    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, double length, double scale, int v, double exp, Func<double, double> f, Func<double, double> df, IEnumerable<int> columnIndices,
+    private static IList<double> GetGradient(double[,] x, int i, int j, double length, double scale, int v, double exp, Func<double, double> f, Func<double, double> df, int[] columnIndices,
       bool fixedLength, bool fixedScale) {
-      double k = Math.Sqrt(Util.SqrDist(x, i, x, j, 1.0 / length, columnIndices));
-      if (!fixedLength) yield return scale * Math.Pow(Math.Max(1.0 - k, 0), exp + v - 1) * k * ((exp + v) * f(k) - Math.Max(1 - k, 0) * df(k));
-      if (!fixedScale) yield return 2.0 * scale * Math.Pow(Math.Max(1 - k, 0), exp + v) * f(k);
+      double k = Math.Sqrt(Util.SqrDist(x, i, x, j, columnIndices, 1.0 / length));
+      var g = new List<double>(2);
+      if (!fixedLength) g.Add(scale * Math.Pow(Math.Max(1.0 - k, 0), exp + v - 1) * k * ((exp + v) * f(k) - Math.Max(1 - k, 0) * df(k)));
+      if (!fixedScale) g.Add(2.0 * scale * Math.Pow(Math.Max(1 - k, 0), exp + v) * f(k));
+      return g;
     }
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePolynomial.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePolynomial.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePolynomial.cs	(revision 14029)
@@ -22,5 +22,4 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -107,5 +106,5 @@
     }
 
-    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
       double @const, scale;
       int degree = DegreeParameter.Value.Value;
@@ -116,15 +115,17 @@
       // create functions
       var cov = new ParameterizedCovarianceFunction();
-      cov.Covariance = (x, i, j) => scale * Math.Pow(@const + Util.ScalarProd(x, i, j, 1.0, columnIndices), degree);
-      cov.CrossCovariance = (x, xt, i, j) => scale * Math.Pow(@const + Util.ScalarProd(x, i, xt, j, 1.0, columnIndices), degree);
+      cov.Covariance = (x, i, j) => scale * Math.Pow(@const + Util.ScalarProd(x, i, j, columnIndices, 1.0), degree);
+      cov.CrossCovariance = (x, xt, i, j) => scale * Math.Pow(@const + Util.ScalarProd(x, i, xt, j, columnIndices, 1.0), degree);
       cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, @const, scale, degree, columnIndices, fixedConst, fixedScale);
       return cov;
     }
 
-    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, double c, double scale, int degree, IEnumerable<int> columnIndices,
+    private static IList<double> GetGradient(double[,] x, int i, int j, double c, double scale, int degree, int[] columnIndices,
       bool fixedConst, bool fixedScale) {
-      double s = Util.ScalarProd(x, i, j, 1.0, columnIndices);
-      if (!fixedConst) yield return c * degree * scale * Math.Pow(c + s, degree - 1);
-      if (!fixedScale) yield return 2 * scale * Math.Pow(c + s, degree);
+      double s = Util.ScalarProd(x, i, j, columnIndices, 1.0);
+      var g = new List<double>(2);
+      if (!fixedConst) g.Add(c * degree * scale * Math.Pow(c + s, degree - 1));
+      if (!fixedScale) g.Add(2 * scale * Math.Pow(c + s, degree));
+      return g;
     }
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceProduct.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceProduct.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceProduct.cs	(revision 14029)
@@ -23,5 +23,4 @@
 using System.Collections.Generic;
 using System.Linq;
-using System.Linq.Expressions;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -76,5 +75,5 @@
     }
 
-    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
       if (factors.Count == 0) throw new ArgumentException("at least one factor is necessary for the product covariance function.");
       var functions = new List<ParameterizedCovarianceFunction>();
@@ -93,6 +92,7 @@
     }
 
-    public static IEnumerable<double> GetGradient(double[,] x, int i, int j, List<ParameterizedCovarianceFunction> factorFunctions) {
+    public static IList<double> GetGradient(double[,] x, int i, int j, List<ParameterizedCovarianceFunction> factorFunctions) {
       var covariances = factorFunctions.Select(f => f.Covariance(x, i, j)).ToArray();
+      var gr = new List<double>();
       for (int ii = 0; ii < factorFunctions.Count; ii++) {
         foreach (var g in factorFunctions[ii].CovarianceGradient(x, i, j)) {
@@ -100,7 +100,8 @@
           for (int jj = 0; jj < covariances.Length; jj++)
             if (ii != jj) res *= covariances[jj];
-          yield return res;
+          gr.Add(res);
         }
       }
+      return gr;
     }
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticArd.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticArd.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticArd.cs	(revision 14029)
@@ -121,5 +121,5 @@
     }
 
-    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
       double scale, shape;
       double[] inverseLength;
@@ -144,5 +144,5 @@
     }
 
-    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, IEnumerable<int> columnIndices, double scale, double shape, double[] inverseLength,
+    private static IList<double> GetGradient(double[,] x, int i, int j, int[] columnIndices, double scale, double shape, double[] inverseLength,
       bool fixedInverseLength, bool fixedScale, bool fixedShape) {
       double d = i == j
@@ -151,14 +151,16 @@
       double b = 1 + 0.5 * d / shape;
       int k = 0;
+      var g = new List<double>(columnIndices.Length + 2);
       if (!fixedInverseLength) {
         foreach (var columnIndex in columnIndices) {
-          yield return
+          g.Add(
             scale * Math.Pow(b, -shape - 1) *
-            Util.SqrDist(x[i, columnIndex] * inverseLength[k], x[j, columnIndex] * inverseLength[k]);
+            Util.SqrDist(x[i, columnIndex] * inverseLength[k], x[j, columnIndex] * inverseLength[k]));
           k++;
         }
       }
-      if (!fixedScale) yield return 2 * scale * Math.Pow(b, -shape);
-      if (!fixedShape) yield return scale * Math.Pow(b, -shape) * (0.5 * d / b - shape * Math.Log(b));
+      if (!fixedScale) g.Add(2 * scale * Math.Pow(b, -shape));
+      if (!fixedShape) g.Add(scale * Math.Pow(b, -shape) * (0.5 * d / b - shape * Math.Log(b)));
+      return g;
     }
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticIso.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticIso.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticIso.cs	(revision 14029)
@@ -117,5 +117,5 @@
     }
 
-    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
       double scale, shape, inverseLength;
       GetParameterValues(p, out scale, out shape, out inverseLength);
@@ -128,9 +128,9 @@
         double d = i == j
                     ? 0.0
-                    : Util.SqrDist(x, i, j, inverseLength, columnIndices);
+                    : Util.SqrDist(x, i, j, columnIndices, inverseLength);
         return scale * Math.Pow(1 + 0.5 * d / shape, -shape);
       };
       cov.CrossCovariance = (x, xt, i, j) => {
-        double d = Util.SqrDist(x, i, xt, j, inverseLength, columnIndices);
+        double d = Util.SqrDist(x, i, xt, j, columnIndices, inverseLength);
         return scale * Math.Pow(1 + 0.5 * d / shape, -shape);
       };
@@ -139,14 +139,16 @@
     }
 
-    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, IEnumerable<int> columnIndices, double scale, double shape, double inverseLength,
+    private static IList<double> GetGradient(double[,] x, int i, int j, int[] columnIndices, double scale, double shape, double inverseLength,
       bool fixedInverseLength, bool fixedScale, bool fixedShape) {
       double d = i == j
                    ? 0.0
-                   : Util.SqrDist(x, i, j, inverseLength, columnIndices);
+                   : Util.SqrDist(x, i, j, columnIndices, inverseLength);
 
       double b = 1 + 0.5 * d / shape;
-      if (!fixedInverseLength) yield return scale * Math.Pow(b, -shape - 1) * d;
-      if (!fixedScale) yield return 2 * scale * Math.Pow(b, -shape);
-      if (!fixedShape) yield return scale * Math.Pow(b, -shape) * (0.5 * d / b - shape * Math.Log(b));
+      var g = new List<double>(3);
+      if (!fixedInverseLength) g.Add(scale * Math.Pow(b, -shape - 1) * d);
+      if (!fixedScale) g.Add(2 * scale * Math.Pow(b, -shape));
+      if (!fixedShape) g.Add(scale * Math.Pow(b, -shape) * (0.5 * d / b - shape * Math.Log(b)));
+      return g;
     }
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceScale.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceScale.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceScale.cs	(revision 14029)
@@ -87,5 +87,5 @@
     }
 
-    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
       double scale;
       GetParameterValues(p, out scale);
@@ -100,11 +100,13 @@
     }
 
-    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, IEnumerable<int> columnIndices, double scale, ParameterizedCovarianceFunction cov,
+    private static IList<double> GetGradient(double[,] x, int i, int j, int[] columnIndices, double scale, ParameterizedCovarianceFunction cov,
       bool fixedScale) {
+      var gr = new List<double>((!fixedScale ? 1 : 0) + cov.CovarianceGradient(x, i, j).Count);
       if (!fixedScale) {
-        yield return 2 * scale * cov.Covariance(x, i, j);
+        gr.Add(2 * scale * cov.Covariance(x, i, j));
       }
       foreach (var g in cov.CovarianceGradient(x, i, j))
-        yield return scale * g;
+        gr.Add(scale * g);
+      return gr;
     }
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSpectralMixture.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSpectralMixture.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSpectralMixture.cs	(revision 14029)
@@ -23,5 +23,4 @@
 using System.Collections.Generic;
 using System.Linq;
-using System.Linq.Expressions;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -131,5 +130,5 @@
     }
 
-    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
       double[] weight, frequency, lengthScale;
       GetParameterValues(p, out weight, out frequency, out lengthScale);
@@ -152,5 +151,5 @@
     }
 
-    private static double GetCovariance(double[,] x, double[,] xt, int i, int j, int maxQ, double[] weight, double[] frequency, double[] lengthScale, IEnumerable<int> columnIndices) {
+    private static double GetCovariance(double[,] x, double[,] xt, int i, int j, int maxQ, double[] weight, double[] frequency, double[] lengthScale, int[] columnIndices) {
       // tau = x - x' (only for selected variables)
       double[] tau =
@@ -164,6 +163,7 @@
         int idx = 0; // helper index for tau
         // for each selected variable
-        foreach (var c in columnIndices) {
-          kc *= f1(tau[idx], lengthScale[q * numberOfVariables + c]) * f2(tau[idx], frequency[q * numberOfVariables + c]);
+        for (int c = 0; c < columnIndices.Length; c++) {
+          var col = columnIndices[c];
+          kc *= f1(tau[idx], lengthScale[q * numberOfVariables + col]) * f2(tau[idx], frequency[q * numberOfVariables + col]);
           idx++;
         }
@@ -181,9 +181,10 @@
 
     // order of returned gradients must match the order in GetParameterValues!
-    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, int maxQ, double[] weight, double[] frequency, double[] lengthScale, IEnumerable<int> columnIndices,
+    private static IList<double> GetGradient(double[,] x, int i, int j, int maxQ, double[] weight, double[] frequency, double[] lengthScale, int[] columnIndices,
       bool fixedWeight, bool fixedFrequency, bool fixedLengthScale) {
       double[] tau = Util.GetRow(x, i, columnIndices).Zip(Util.GetRow(x, j, columnIndices), (xi, xj) => xi - xj).ToArray();
       int numberOfVariables = lengthScale.Length / maxQ;
 
+      var g = new List<double>((!fixedWeight ? maxQ : 0) + (!fixedFrequency ? maxQ * columnIndices.Length : 0) + (!fixedLengthScale ? maxQ * columnIndices.Length : 0));
       if (!fixedWeight) {
         // weight
@@ -193,9 +194,10 @@
           int idx = 0; // helper index for tau
           // for each selected variable
-          foreach (var c in columnIndices) {
-            k *= f1(tau[idx], lengthScale[q * numberOfVariables + c]) * f2(tau[idx], frequency[q * numberOfVariables + c]);
+          for (int c = 0; c < columnIndices.Length; c++) {
+            var col = columnIndices[c];
+            k *= f1(tau[idx], lengthScale[q * numberOfVariables + col]) * f2(tau[idx], frequency[q * numberOfVariables + col]);
             idx++;
           }
-          yield return k;
+          g.Add(k);
         }
       }
@@ -212,5 +214,5 @@
                        Math.Sin(2 * Math.PI * tau[idx] * frequency[q * numberOfVariables + c]);
             idx++;
-            yield return weight[q] * k;
+            g.Add(weight[q] * k);
           }
         }
@@ -228,8 +230,10 @@
                        f2(tau[idx], frequency[q * numberOfVariables + c]);
             idx++;
-            yield return weight[q] * k;
+            g.Add(weight[q] * k);
           }
         }
       }
+
+      return g;
     }
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialArd.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialArd.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialArd.cs	(revision 14029)
@@ -99,5 +99,5 @@
     }
 
-    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
       double scale;
       double[] inverseLength;
@@ -122,5 +122,5 @@
 
     // order of returned gradients must match the order in GetParameterValues!
-    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, IEnumerable<int> columnIndices, double scale, double[] inverseLength,
+    private static IList<double> GetGradient(double[,] x, int i, int j, int[] columnIndices, double scale, double[] inverseLength,
       bool fixedInverseLength, bool fixedScale) {
       double d = i == j
@@ -129,12 +129,15 @@
 
       int k = 0;
+      var g = new List<double>((!fixedInverseLength ? columnIndices.Length : 0) + (!fixedScale ? 1 : 0));
       if (!fixedInverseLength) {
-        foreach (var columnIndex in columnIndices) {
+        for (int c = 0; c < columnIndices.Length; c++) {
+          var columnIndex = columnIndices[c];
           double sqrDist = Util.SqrDist(x[i, columnIndex] * inverseLength[k], x[j, columnIndex] * inverseLength[k]);
-          yield return scale * Math.Exp(-d / 2.0) * sqrDist;
+          g.Add(scale * Math.Exp(-d / 2.0) * sqrDist);
           k++;
         }
       }
-      if (!fixedScale) yield return 2.0 * scale * Math.Exp(-d / 2.0);
+      if (!fixedScale) g.Add(2.0 * scale * Math.Exp(-d / 2.0));
+      return g;
     }
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialIso.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialIso.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialIso.cs	(revision 14029)
@@ -22,5 +22,4 @@
 using System;
 using System.Collections.Generic;
-using System.Linq.Expressions;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -104,5 +103,5 @@
     }
 
-    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
       double inverseLength, scale;
       GetParameterValues(p, out scale, out inverseLength);
@@ -114,9 +113,9 @@
         double d = i == j
                 ? 0.0
-                : Util.SqrDist(x, i, j, inverseLength, columnIndices);
+                : Util.SqrDist(x, i, j, columnIndices, inverseLength);
         return scale * Math.Exp(-d / 2.0);
       };
       cov.CrossCovariance = (x, xt, i, j) => {
-        double d = Util.SqrDist(x, i, xt, j, inverseLength, columnIndices);
+        double d = Util.SqrDist(x, i, xt, j, columnIndices, inverseLength);
         return scale * Math.Exp(-d / 2.0);
       };
@@ -127,12 +126,14 @@
 
     // order of returned gradients must match the order in GetParameterValues!
-    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, double sf2, double inverseLength, IEnumerable<int> columnIndices, 
+    private static IList<double> GetGradient(double[,] x, int i, int j, double sf2, double inverseLength, int[] columnIndices,
       bool fixedInverseLength, bool fixedScale) {
       double d = i == j
                    ? 0.0
-                   : Util.SqrDist(x, i, j, inverseLength, columnIndices);
+                   : Util.SqrDist(x, i, j, columnIndices, inverseLength);
       double g = Math.Exp(-d / 2.0);
-      if (!fixedInverseLength) yield return sf2 * g * d;
-      if (!fixedScale) yield return 2.0 * sf2 * g;
+      var gr = new List<double>(2);
+      if (!fixedInverseLength) gr.Add(sf2 * g * d);
+      if (!fixedScale) gr.Add(2.0 * sf2 * g);
+      return gr;
     }
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSum.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSum.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSum.cs	(revision 14029)
@@ -23,5 +23,4 @@
 using System.Collections.Generic;
 using System.Linq;
-using System.Linq.Expressions;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -76,5 +75,5 @@
     }
 
-    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
       if (terms.Count == 0) throw new ArgumentException("at least one term is necessary for the product covariance function.");
       var functions = new List<ParameterizedCovarianceFunction>();
@@ -88,5 +87,10 @@
       sum.Covariance = (x, i, j) => functions.Select(e => e.Covariance(x, i, j)).Sum();
       sum.CrossCovariance = (x, xt, i, j) => functions.Select(e => e.CrossCovariance(x, xt, i, j)).Sum();
-      sum.CovarianceGradient = (x, i, j) => functions.Select(e => e.CovarianceGradient(x, i, j)).Aggregate(Enumerable.Concat);
+      sum.CovarianceGradient = (x, i, j) => {
+        var g = new List<double>();
+        foreach (var e in functions)
+          g.AddRange(e.CovarianceGradient(x, i, j));
+        return g;
+      };
       return sum;
     }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessBase.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessBase.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessBase.cs	(revision 14029)
@@ -21,6 +21,4 @@
 #endregion
 
-using System;
-using System.Linq;
 using HeuristicLab.Algorithms.GradientDescent;
 using HeuristicLab.Common;
@@ -31,5 +29,4 @@
 using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using HeuristicLab.PluginInfrastructure;
 using HeuristicLab.Problems.DataAnalysis;
 
@@ -51,4 +48,5 @@
     protected const string HyperparameterGradientsParameterName = "HyperparameterGradients";
     protected const string SolutionCreatorParameterName = "GaussianProcessSolutionCreator";
+    protected const string ScaleInputValuesParameterName = "ScaleInputValues";
 
     public new IDataAnalysisProblem Problem {
@@ -72,4 +70,7 @@
     public IValueParameter<BoolValue> SetSeedRandomlyParameter {
       get { return (IValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> ScaleInputValuesParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[ScaleInputValuesParameterName]; }
     }
     #endregion
@@ -89,4 +90,9 @@
     public int Seed { get { return SeedParameter.Value.Value; } set { SeedParameter.Value.Value = value; } }
     public bool SetSeedRandomly { get { return SetSeedRandomlyParameter.Value.Value; } set { SetSeedRandomlyParameter.Value.Value = value; } }
+
+    public bool ScaleInputValues {
+      get { return ScaleInputValuesParameter.Value.Value; }
+      set { ScaleInputValuesParameter.Value.Value = value; }
+    }
     #endregion
 
@@ -107,4 +113,8 @@
       Parameters.Add(new ValueParameter<BoolValue>(ApproximateGradientsParameterName, "Indicates that gradients should not be approximated (necessary for LM-BFGS).", new BoolValue(false)));
       Parameters[ApproximateGradientsParameterName].Hidden = true; // should not be changed
+
+      Parameters.Add(new FixedValueParameter<BoolValue>(ScaleInputValuesParameterName,
+        "Determines if the input variable values are scaled to the range [0..1] for training.", new BoolValue(true)));
+      Parameters[ScaleInputValuesParameterName].Hidden = true;
 
       // necessary for BFGS
@@ -191,4 +201,10 @@
         Parameters["Maximization"].Hidden = true;
       }
+
+      if (!Parameters.ContainsKey(ScaleInputValuesParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(ScaleInputValuesParameterName,
+          "Determines if the input variable values are scaled to the range [0..1] for training.", new BoolValue(true)));
+        Parameters[ScaleInputValuesParameterName].Hidden = true;
+      }
       #endregion
     }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassification.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassification.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassification.cs	(revision 14029)
@@ -25,4 +25,5 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Data;
 using HeuristicLab.Optimization;
 using HeuristicLab.Parameters;
@@ -48,4 +49,5 @@
 
     private const string ModelParameterName = "Model";
+    private const string CreateSolutionParameterName = "CreateSolution";
 
     #region parameter properties
@@ -55,4 +57,13 @@
     public IFixedValueParameter<GaussianProcessClassificationSolutionCreator> GaussianProcessSolutionCreatorParameter {
       get { return (IFixedValueParameter<GaussianProcessClassificationSolutionCreator>)Parameters[SolutionCreatorParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> CreateSolutionParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
+    }
+    #endregion
+    #region properties
+    public bool CreateSolution {
+      get { return CreateSolutionParameter.Value.Value; }
+      set { CreateSolutionParameter.Value.Value = value; }
     }
     #endregion
@@ -82,4 +93,7 @@
         new GaussianProcessClassificationSolutionCreator()));
       Parameters[SolutionCreatorParameterName].Hidden = true;
+      // TODO: it would be better to deactivate the solution creator when this parameter is changed
+      Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+      Parameters[CreateSolutionParameterName].Hidden = true;
 
       ParameterizedModelCreators();
@@ -91,4 +105,11 @@
     [StorableHook(HookType.AfterDeserialization)]
     private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
+      if (!Parameters.ContainsKey(CreateSolutionParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+        Parameters[CreateSolutionParameterName].Hidden = true;
+      }
+      #endregion
       RegisterEventHandlers();
     }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationModelCreator.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationModelCreator.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationModelCreator.cs	(revision 14029)
@@ -62,12 +62,10 @@
     public override IOperation Apply() {
       try {
-        var model = Create(ProblemData, Hyperparameter.ToArray(), MeanFunction, CovarianceFunction);
+        var model = Create(ProblemData, Hyperparameter.ToArray(), MeanFunction, CovarianceFunction, ScaleInputValues);
         ModelParameter.ActualValue = model;
         NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(model.NegativeLogLikelihood);
         HyperparameterGradientsParameter.ActualValue = new RealVector(model.HyperparameterGradients);
         return base.Apply();
-      }
-      catch (ArgumentException) { }
-      catch (alglib.alglibexception) { }
+      } catch (ArgumentException) { } catch (alglib.alglibexception) { }
       NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(1E300);
       HyperparameterGradientsParameter.ActualValue = new RealVector(Hyperparameter.Count());
@@ -75,6 +73,6 @@
     }
 
-    public static IGaussianProcessModel Create(IClassificationProblemData problemData, double[] hyperparameter, IMeanFunction meanFunction, ICovarianceFunction covarianceFunction) {
-      return new GaussianProcessModel(problemData.Dataset, problemData.TargetVariable, problemData.AllowedInputVariables, problemData.TrainingIndices, hyperparameter, meanFunction, covarianceFunction);
+    public static IGaussianProcessModel Create(IClassificationProblemData problemData, double[] hyperparameter, IMeanFunction meanFunction, ICovarianceFunction covarianceFunction, bool scaleInputs = true) {
+      return new GaussianProcessModel(problemData.Dataset, problemData.TargetVariable, problemData.AllowedInputVariables, problemData.TrainingIndices, hyperparameter, meanFunction, covarianceFunction, scaleInputs);
     }
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationSolutionCreator.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationSolutionCreator.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationSolutionCreator.cs	(revision 14029)
@@ -40,4 +40,5 @@
     private const string TrainingAccuracyResultName = "Accuracy (training)";
     private const string TestAccuracyResultName = "Accuracy (test)";
+    private const string CreateSolutionParameterName = "CreateSolution";
 
     #region Parameter Properties
@@ -54,4 +55,7 @@
       get { return (ILookupParameter<ResultCollection>)Parameters[ResultsParameterName]; }
     }
+    public ILookupParameter<BoolValue> CreateSolutionParameter {
+      get { return (ILookupParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
+    }
     #endregion
 
@@ -64,8 +68,20 @@
       Parameters.Add(new LookupParameter<IClassificationProblemData>(ProblemDataParameterName, "The classification problem data for the Gaussian process solution."));
       Parameters.Add(new LookupParameter<IGaussianProcessModel>(ModelParameterName, "The Gaussian process classification model to use for the solution."));
+      Parameters.Add(new LookupParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run"));
+
       // in & out
       Parameters.Add(new LookupParameter<ResultCollection>(ResultsParameterName, "The result collection of the algorithm."));
       // out
       Parameters.Add(new LookupParameter<IDiscriminantFunctionClassificationSolution>(SolutionParameterName, "The produced Gaussian process solution."));
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
+      if (!Parameters.ContainsKey(CreateSolutionParameterName)) {
+        Parameters.Add(new LookupParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run"));
+      }
+      #endregion
     }
 
@@ -75,5 +91,5 @@
 
     public override IOperation Apply() {
-      if (ModelParameter.ActualValue != null) {
+      if (ModelParameter.ActualValue != null && CreateSolutionParameter.ActualValue.Value == true) {
         var m = (IGaussianProcessModel)ModelParameter.ActualValue.Clone();
         m.FixParameters();
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessCovarianceOptimizationProblem.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessCovarianceOptimizationProblem.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessCovarianceOptimizationProblem.cs	(revision 14029)
@@ -0,0 +1,424 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.Instances;
+
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("Gaussian Process Covariance Optimization Problem", "")]
+  [Creatable(CreatableAttribute.Categories.GeneticProgrammingProblems, Priority = 300)]
+  [StorableClass]
+  public sealed class GaussianProcessCovarianceOptimizationProblem : SymbolicExpressionTreeProblem, IStatefulItem, IRegressionProblem, IProblemInstanceConsumer<IRegressionProblemData>, IProblemInstanceExporter<IRegressionProblemData> {
+    #region static variables and ctor
+    private static readonly CovarianceMaternIso maternIso1;
+    private static readonly CovarianceMaternIso maternIso3;
+    private static readonly CovarianceMaternIso maternIso5;
+    private static readonly CovariancePiecewisePolynomial piecewisePoly0;
+    private static readonly CovariancePiecewisePolynomial piecewisePoly1;
+    private static readonly CovariancePiecewisePolynomial piecewisePoly2;
+    private static readonly CovariancePiecewisePolynomial piecewisePoly3;
+    private static readonly CovariancePolynomial poly2;
+    private static readonly CovariancePolynomial poly3;
+    private static readonly CovarianceSpectralMixture spectralMixture1;
+    private static readonly CovarianceSpectralMixture spectralMixture3;
+    private static readonly CovarianceSpectralMixture spectralMixture5;
+    private static readonly CovarianceLinear linear;
+    private static readonly CovarianceLinearArd linearArd;
+    private static readonly CovarianceNeuralNetwork neuralNetwork;
+    private static readonly CovariancePeriodic periodic;
+    private static readonly CovarianceRationalQuadraticIso ratQuadraticIso;
+    private static readonly CovarianceRationalQuadraticArd ratQuadraticArd;
+    private static readonly CovarianceSquaredExponentialArd sqrExpArd;
+    private static readonly CovarianceSquaredExponentialIso sqrExpIso;
+
+    static GaussianProcessCovarianceOptimizationProblem() {
+      // cumbersome initialization because of ConstrainedValueParameters
+      maternIso1 = new CovarianceMaternIso(); SetConstrainedValueParameter(maternIso1.DParameter, 1);
+      maternIso3 = new CovarianceMaternIso(); SetConstrainedValueParameter(maternIso3.DParameter, 3);
+      maternIso5 = new CovarianceMaternIso(); SetConstrainedValueParameter(maternIso5.DParameter, 5);
+
+      piecewisePoly0 = new CovariancePiecewisePolynomial(); SetConstrainedValueParameter(piecewisePoly0.VParameter, 0);
+      piecewisePoly1 = new CovariancePiecewisePolynomial(); SetConstrainedValueParameter(piecewisePoly1.VParameter, 1);
+      piecewisePoly2 = new CovariancePiecewisePolynomial(); SetConstrainedValueParameter(piecewisePoly2.VParameter, 2);
+      piecewisePoly3 = new CovariancePiecewisePolynomial(); SetConstrainedValueParameter(piecewisePoly3.VParameter, 3);
+
+      poly2 = new CovariancePolynomial(); poly2.DegreeParameter.Value.Value = 2;
+      poly3 = new CovariancePolynomial(); poly3.DegreeParameter.Value.Value = 3;
+
+      spectralMixture1 = new CovarianceSpectralMixture(); spectralMixture1.QParameter.Value.Value = 1;
+      spectralMixture3 = new CovarianceSpectralMixture(); spectralMixture3.QParameter.Value.Value = 3;
+      spectralMixture5 = new CovarianceSpectralMixture(); spectralMixture5.QParameter.Value.Value = 5;
+
+      linear = new CovarianceLinear();
+      linearArd = new CovarianceLinearArd();
+      neuralNetwork = new CovarianceNeuralNetwork();
+      periodic = new CovariancePeriodic();
+      ratQuadraticArd = new CovarianceRationalQuadraticArd();
+      ratQuadraticIso = new CovarianceRationalQuadraticIso();
+      sqrExpArd = new CovarianceSquaredExponentialArd();
+      sqrExpIso = new CovarianceSquaredExponentialIso();
+    }
+
+    private static void SetConstrainedValueParameter(IConstrainedValueParameter<IntValue> param, int val) {
+      param.Value = param.ValidValues.Single(v => v.Value == val);
+    }
+
+    #endregion
+
+    #region parameter names
+
+    private const string ProblemDataParameterName = "ProblemData";
+    private const string ConstantOptIterationsParameterName = "Constant optimization steps";
+    private const string RestartsParameterName = "Restarts";
+
+    #endregion
+
+    #region Parameter Properties
+    IParameter IDataAnalysisProblem.ProblemDataParameter { get { return ProblemDataParameter; } }
+
+    public IValueParameter<IRegressionProblemData> ProblemDataParameter {
+      get { return (IValueParameter<IRegressionProblemData>)Parameters[ProblemDataParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> ConstantOptIterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[ConstantOptIterationsParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> RestartsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[RestartsParameterName]; }
+    }
+    #endregion
+
+    #region Properties
+
+    public IRegressionProblemData ProblemData {
+      get { return ProblemDataParameter.Value; }
+      set { ProblemDataParameter.Value = value; }
+    }
+    IDataAnalysisProblemData IDataAnalysisProblem.ProblemData { get { return ProblemData; } }
+
+    public int ConstantOptIterations {
+      get { return ConstantOptIterationsParameter.Value.Value; }
+      set { ConstantOptIterationsParameter.Value.Value = value; }
+    }
+
+    public int Restarts {
+      get { return RestartsParameter.Value.Value; }
+      set { RestartsParameter.Value.Value = value; }
+    }
+    #endregion
+
+    public override bool Maximization {
+      get { return true; } // return log likelihood (instead of negative log likelihood as in GPR
+    }
+
+    // problem stores a few variables for information exchange from Evaluate() to Analyze()
+    private readonly object problemStateLocker = new object();
+    [Storable]
+    private double bestQ;
+    [Storable]
+    private double[] bestHyperParameters;
+    [Storable]
+    private IMeanFunction meanFunc;
+    [Storable]
+    private ICovarianceFunction covFunc;
+
+    public GaussianProcessCovarianceOptimizationProblem()
+      : base() {
+      Parameters.Add(new ValueParameter<IRegressionProblemData>(ProblemDataParameterName, "The data for the regression problem", new RegressionProblemData()));
+      Parameters.Add(new FixedValueParameter<IntValue>(ConstantOptIterationsParameterName, "Number of optimization steps for hyperparameter values", new IntValue(50)));
+      Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of random restarts for constant optimization.", new IntValue(10)));
+      Parameters["Restarts"].Hidden = true;
+      var g = new SimpleSymbolicExpressionGrammar();
+      g.AddSymbols(new string[] { "Sum", "Product" }, 2, 2);
+      g.AddTerminalSymbols(new string[]
+      {
+        "Linear",
+        "LinearArd",
+        "MaternIso1",
+        "MaternIso3",
+        "MaternIso5",
+        "NeuralNetwork",
+        "Periodic",
+        "PiecewisePolynomial0",
+        "PiecewisePolynomial1",
+        "PiecewisePolynomial2",
+        "PiecewisePolynomial3",
+        "Polynomial2",
+        "Polynomial3",
+        "RationalQuadraticArd",
+        "RationalQuadraticIso",
+        "SpectralMixture1",
+        "SpectralMixture3",
+        "SpectralMixture5",
+        "SquaredExponentialArd",
+        "SquaredExponentialIso"
+      });
+      base.Encoding = new SymbolicExpressionTreeEncoding(g, 10, 5);
+    }
+
+    public void InitializeState() { ClearState(); }
+    public void ClearState() {
+      meanFunc = null;
+      covFunc = null;
+      bestQ = double.NegativeInfinity;
+      bestHyperParameters = null;
+    }
+
+    private readonly object syncRoot = new object();
+    // Does not produce the same result for the same seed when using parallel engine (see below)!
+    public override double Evaluate(ISymbolicExpressionTree tree, IRandom random) {
+      var meanFunction = new MeanConst();
+      var problemData = ProblemData;
+      var ds = problemData.Dataset;
+      var targetVariable = problemData.TargetVariable;
+      var allowedInputVariables = problemData.AllowedInputVariables.ToArray();
+      var nVars = allowedInputVariables.Length;
+      var trainingRows = problemData.TrainingIndices.ToArray();
+
+      // use the same covariance function for each restart
+      var covarianceFunction = TreeToCovarianceFunction(tree);
+
+      // allocate hyperparameters
+      var hyperParameters = new double[meanFunction.GetNumberOfParameters(nVars) + covarianceFunction.GetNumberOfParameters(nVars) + 1]; // mean + cov + noise
+      double[] bestHyperParameters = new double[hyperParameters.Length];
+      var bestObjValue = new double[1] { double.MinValue };
+
+      // data that is necessary for the objective function
+      var data = Tuple.Create(ds, targetVariable, allowedInputVariables, trainingRows, (IMeanFunction)meanFunction, covarianceFunction, bestObjValue);
+
+      for (int t = 0; t < Restarts; t++) {
+        var prevBest = bestObjValue[0];
+        var prevBestHyperParameters = new double[hyperParameters.Length];
+        Array.Copy(bestHyperParameters, prevBestHyperParameters, bestHyperParameters.Length);
+
+        // initialize hyperparameters
+        hyperParameters[0] = ds.GetDoubleValues(targetVariable).Average(); // mean const
+
+        // Evaluate might be called concurrently therefore access to random has to be synchronized.
+        // However, results of multiple runs with the same seed will be different when using the parallel engine.
+        lock (syncRoot) {
+          for (int i = 0; i < covarianceFunction.GetNumberOfParameters(nVars); i++) {
+            hyperParameters[1 + i] = random.NextDouble() * 2.0 - 1.0;
+          }
+        }
+        hyperParameters[hyperParameters.Length - 1] = 1.0; // s² = exp(2), TODO: other inits better?
+
+        // use alglib.bfgs for hyper-parameter optimization ...
+        double epsg = 0;
+        double epsf = 0.00001;
+        double epsx = 0;
+        double stpmax = 1;
+        int maxits = ConstantOptIterations;
+        alglib.mincgstate state;
+        alglib.mincgreport rep;
+
+        alglib.mincgcreate(hyperParameters, out state);
+        alglib.mincgsetcond(state, epsg, epsf, epsx, maxits);
+        alglib.mincgsetstpmax(state, stpmax);
+        alglib.mincgoptimize(state, ObjectiveFunction, null, data);
+
+        alglib.mincgresults(state, out bestHyperParameters, out rep);
+
+        if (rep.terminationtype < 0) {
+          // error -> restore previous best quality
+          bestObjValue[0] = prevBest;
+          Array.Copy(prevBestHyperParameters, bestHyperParameters, prevBestHyperParameters.Length);
+        }
+      }
+
+      UpdateBestSoFar(bestObjValue[0], bestHyperParameters, meanFunction, covarianceFunction);
+
+      return bestObjValue[0];
+    }
+
+    // updates the overall best quality and overall best model for Analyze()
+    private void UpdateBestSoFar(double bestQ, double[] bestHyperParameters, IMeanFunction meanFunc, ICovarianceFunction covFunc) {
+      lock (problemStateLocker) {
+        if (bestQ > this.bestQ) {
+          this.bestQ = bestQ;
+          this.bestHyperParameters = new double[bestHyperParameters.Length];
+          Array.Copy(bestHyperParameters, this.bestHyperParameters, this.bestHyperParameters.Length);
+          this.meanFunc = meanFunc;
+          this.covFunc = covFunc;
+        }
+      }
+    }
+
+    public override void Analyze(ISymbolicExpressionTree[] trees, double[] qualities, ResultCollection results, IRandom random) {
+      if (!results.ContainsKey("Best Solution Quality")) {
+        results.Add(new Result("Best Solution Quality", typeof(DoubleValue)));
+      }
+      if (!results.ContainsKey("Best Tree")) {
+        results.Add(new Result("Best Tree", typeof(ISymbolicExpressionTree)));
+      }
+      if (!results.ContainsKey("Best Solution")) {
+        results.Add(new Result("Best Solution", typeof(GaussianProcessRegressionSolution)));
+      }
+
+      var bestQuality = qualities.Max();
+
+      if (results["Best Solution Quality"].Value == null || bestQuality > ((DoubleValue)results["Best Solution Quality"].Value).Value) {
+        var bestIdx = Array.IndexOf(qualities, bestQuality);
+        var bestClone = (ISymbolicExpressionTree)trees[bestIdx].Clone();
+        results["Best Tree"].Value = bestClone;
+        results["Best Solution Quality"].Value = new DoubleValue(bestQuality);
+        results["Best Solution"].Value = CreateSolution();
+      }
+    }
+
+    private IItem CreateSolution() {
+      var problemData = ProblemData;
+      var ds = problemData.Dataset;
+      var targetVariable = problemData.TargetVariable;
+      var allowedInputVariables = problemData.AllowedInputVariables.ToArray();
+      var trainingRows = problemData.TrainingIndices.ToArray();
+
+      lock (problemStateLocker) {
+        var model = new GaussianProcessModel(ds, targetVariable, allowedInputVariables, trainingRows, bestHyperParameters, (IMeanFunction)meanFunc.Clone(), (ICovarianceFunction)covFunc.Clone());
+        model.FixParameters();
+        return model.CreateRegressionSolution((IRegressionProblemData)ProblemData.Clone());
+      }
+    }
+
+    private void ObjectiveFunction(double[] x, ref double func, double[] grad, object obj) {
+      // we want to optimize the model likelihood by changing the hyperparameters and also return the gradient for each hyperparameter
+      var data = (Tuple<IDataset, string, string[], int[], IMeanFunction, ICovarianceFunction, double[]>)obj;
+      var ds = data.Item1;
+      var targetVariable = data.Item2;
+      var allowedInputVariables = data.Item3;
+      var trainingRows = data.Item4;
+      var meanFunction = data.Item5;
+      var covarianceFunction = data.Item6;
+      var bestObjValue = data.Item7;
+      var hyperParameters = x; // the decision variable vector
+
+      try {
+        var model = new GaussianProcessModel(ds, targetVariable, allowedInputVariables, trainingRows, hyperParameters, meanFunction, covarianceFunction);
+
+        func = model.NegativeLogLikelihood; // mincgoptimize, so we return negative likelihood
+        bestObjValue[0] = Math.Max(bestObjValue[0], -func); // problem itself is a maximization problem
+        var gradients = model.HyperparameterGradients;
+        Array.Copy(gradients, grad, gradients.Length);
+      }
+      catch (ArgumentException) {
+        // building the GaussianProcessModel might fail, in this case we return the worst possible objective value
+        func = 1.0E+300;
+        Array.Clear(grad, 0, grad.Length);
+      }
+    }
+
+    private ICovarianceFunction TreeToCovarianceFunction(ISymbolicExpressionTree tree) {
+      return TreeToCovarianceFunction(tree.Root.GetSubtree(0).GetSubtree(0)); // skip programroot and startsymbol
+    }
+
+    private ICovarianceFunction TreeToCovarianceFunction(ISymbolicExpressionTreeNode node) {
+      switch (node.Symbol.Name) {
+        case "Sum": {
+            var sum = new CovarianceSum();
+            sum.Terms.Add(TreeToCovarianceFunction(node.GetSubtree(0)));
+            sum.Terms.Add(TreeToCovarianceFunction(node.GetSubtree(1)));
+            return sum;
+          }
+        case "Product": {
+            var prod = new CovarianceProduct();
+            prod.Factors.Add(TreeToCovarianceFunction(node.GetSubtree(0)));
+            prod.Factors.Add(TreeToCovarianceFunction(node.GetSubtree(1)));
+            return prod;
+          }
+        // covFunction is cloned by the model so we can reuse instances of terminal covariance functions
+        case "Linear": return linear;
+        case "LinearArd": return linearArd;
+        case "MaternIso1": return maternIso1;
+        case "MaternIso3": return maternIso3;
+        case "MaternIso5": return maternIso5;
+        case "NeuralNetwork": return neuralNetwork;
+        case "Periodic": return periodic;
+        case "PiecewisePolynomial0": return piecewisePoly0;
+        case "PiecewisePolynomial1": return piecewisePoly1;
+        case "PiecewisePolynomial2": return piecewisePoly2;
+        case "PiecewisePolynomial3": return piecewisePoly3;
+        case "Polynomial2": return poly2;
+        case "Polynomial3": return poly3;
+        case "RationalQuadraticArd": return ratQuadraticArd;
+        case "RationalQuadraticIso": return ratQuadraticIso;
+        case "SpectralMixture1": return spectralMixture1;
+        case "SpectralMixture3": return spectralMixture3;
+        case "SpectralMixture5": return spectralMixture5;
+        case "SquaredExponentialArd": return sqrExpArd;
+        case "SquaredExponentialIso": return sqrExpIso;
+        default: throw new InvalidProgramException(string.Format("Found invalid symbol {0}", node.Symbol.Name));
+      }
+    }
+
+
+    // persistence
+    [StorableConstructor]
+    private GaussianProcessCovarianceOptimizationProblem(bool deserializing) : base(deserializing) { }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+    }
+
+    // cloning 
+    private GaussianProcessCovarianceOptimizationProblem(GaussianProcessCovarianceOptimizationProblem original, Cloner cloner)
+      : base(original, cloner) {
+      bestQ = original.bestQ;
+      meanFunc = cloner.Clone(original.meanFunc);
+      covFunc = cloner.Clone(original.covFunc);
+      if (bestHyperParameters != null) {
+        bestHyperParameters = new double[original.bestHyperParameters.Length];
+        Array.Copy(original.bestHyperParameters, bestHyperParameters, bestHyperParameters.Length);
+      }
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessCovarianceOptimizationProblem(this, cloner);
+    }
+
+    public void Load(IRegressionProblemData data) {
+      this.ProblemData = data;
+      OnProblemDataChanged();
+    }
+
+    public IRegressionProblemData Export() {
+      return ProblemData;
+    }
+
+    #region events
+    public event EventHandler ProblemDataChanged;
+
+
+    private void OnProblemDataChanged() {
+      var handler = ProblemDataChanged;
+      if (handler != null)
+        handler(this, EventArgs.Empty);
+    }
+    #endregion
+
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs	(revision 14029)
@@ -34,5 +34,9 @@
   [StorableClass]
   [Item("GaussianProcessModel", "Represents a Gaussian process posterior.")]
-  public sealed class GaussianProcessModel : NamedItem, IGaussianProcessModel {
+  public sealed class GaussianProcessModel : RegressionModel, IGaussianProcessModel {
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return allowedInputVariables; }
+    }
+
     [Storable]
     private double negativeLogLikelihood;
@@ -61,9 +65,5 @@
       get { return meanFunction; }
     }
-    [Storable]
-    private string targetVariable;
-    public string TargetVariable {
-      get { return targetVariable; }
-    }
+
     [Storable]
     private string[] allowedInputVariables;
@@ -124,8 +124,8 @@
       this.meanFunction = cloner.Clone(original.meanFunction);
       this.covarianceFunction = cloner.Clone(original.covarianceFunction);
-      this.inputScaling = cloner.Clone(original.inputScaling);
+      if (original.inputScaling != null)
+        this.inputScaling = cloner.Clone(original.inputScaling);
       this.trainingDataset = cloner.Clone(original.trainingDataset);
       this.negativeLogLikelihood = original.negativeLogLikelihood;
-      this.targetVariable = original.targetVariable;
       this.sqrSigmaNoise = original.sqrSigmaNoise;
       if (original.meanParameter != null) {
@@ -144,11 +144,11 @@
     }
     public GaussianProcessModel(IDataset ds, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<int> rows,
-      IEnumerable<double> hyp, IMeanFunction meanFunction, ICovarianceFunction covarianceFunction)
-      : base() {
+      IEnumerable<double> hyp, IMeanFunction meanFunction, ICovarianceFunction covarianceFunction,
+      bool scaleInputs = true)
+      : base(targetVariable) {
       this.name = ItemName;
       this.description = ItemDescription;
       this.meanFunction = (IMeanFunction)meanFunction.Clone();
       this.covarianceFunction = (ICovarianceFunction)covarianceFunction.Clone();
-      this.targetVariable = targetVariable;
       this.allowedInputVariables = allowedInputVariables.ToArray();
 
@@ -163,27 +163,35 @@
                                              .ToArray();
       sqrSigmaNoise = Math.Exp(2.0 * hyp.Last());
-      CalculateModel(ds, rows);
-    }
-
-    private void CalculateModel(IDataset ds, IEnumerable<int> rows) {
+      try {
+        CalculateModel(ds, rows, scaleInputs);
+      }
+      catch (alglib.alglibexception ae) {
+        // wrap exception so that calling code doesn't have to know about alglib implementation
+        throw new ArgumentException("There was a problem in the calculation of the Gaussian process model", ae);
+      }
+    }
+
+    private void CalculateModel(IDataset ds, IEnumerable<int> rows, bool scaleInputs = true) {
       this.trainingDataset = (IDataset)ds.Clone();
       this.trainingRows = rows.ToArray();
-      this.inputScaling = new Scaling(trainingDataset, allowedInputVariables, rows);
-      this.x = CalculateX(trainingDataset, allowedInputVariables, rows, inputScaling);
-      var y = ds.GetDoubleValues(targetVariable, rows);
+      this.inputScaling = scaleInputs ? new Scaling(ds, allowedInputVariables, rows) : null;
+
+      x = GetData(ds, this.allowedInputVariables, this.trainingRows, this.inputScaling);
+
+      IEnumerable<double> y;
+      y = ds.GetDoubleValues(TargetVariable, rows);
 
       int n = x.GetLength(0);
 
+      var columns = Enumerable.Range(0, x.GetLength(1)).ToArray();
       // calculate cholesky decomposed (lower triangular) covariance matrix
-      var cov = covarianceFunction.GetParameterizedCovarianceFunction(covarianceParameter, Enumerable.Range(0, x.GetLength(1)));
+      var cov = covarianceFunction.GetParameterizedCovarianceFunction(covarianceParameter, columns);
       this.l = CalculateL(x, cov, sqrSigmaNoise);
 
       // calculate mean
-      var mean = meanFunction.GetParameterizedMeanFunction(meanParameter, Enumerable.Range(0, x.GetLength(1)));
+      var mean = meanFunction.GetParameterizedMeanFunction(meanParameter, columns);
       double[] m = Enumerable.Range(0, x.GetLength(0))
         .Select(r => mean.Mean(x, r))
         .ToArray();
-
-
 
       // calculate sum of diagonal elements for likelihood
@@ -219,6 +227,7 @@
       double[] meanGradients = new double[meanFunction.GetNumberOfParameters(nAllowedVariables)];
       for (int k = 0; k < meanGradients.Length; k++) {
-        var meanGrad = Enumerable.Range(0, alpha.Length)
-        .Select(r => mean.Gradient(x, r, k));
+        var meanGrad = new double[alpha.Length];
+        for (int g = 0; g < meanGrad.Length; g++)
+          meanGrad[g] = mean.Gradient(x, g, k);
         meanGradients[k] = -Util.ScalarProd(meanGrad, alpha);
       }
@@ -228,5 +237,5 @@
         for (int i = 0; i < n; i++) {
           for (int j = 0; j < i; j++) {
-            var g = cov.CovarianceGradient(x, i, j).ToArray();
+            var g = cov.CovarianceGradient(x, i, j);
             for (int k = 0; k < covGradients.Length; k++) {
               covGradients[k] += lCopy[i, j] * g[k];
@@ -234,5 +243,5 @@
           }
 
-          var gDiag = cov.CovarianceGradient(x, i, i).ToArray();
+          var gDiag = cov.CovarianceGradient(x, i, i);
           for (int k = 0; k < covGradients.Length; k++) {
             // diag
@@ -249,6 +258,10 @@
     }
 
-    private static double[,] CalculateX(IDataset ds, IEnumerable<string> allowedInputVariables, IEnumerable<int> rows, Scaling inputScaling) {
-      return AlglibUtil.PrepareAndScaleInputMatrix(ds, allowedInputVariables, rows, inputScaling);
+    private static double[,] GetData(IDataset ds, IEnumerable<string> allowedInputs, IEnumerable<int> rows, Scaling scaling) {
+      if (scaling != null) {
+        return AlglibUtil.PrepareAndScaleInputMatrix(ds, allowedInputs, rows, scaling);
+      } else {
+        return AlglibUtil.PrepareInputMatrix(ds, allowedInputs, rows);
+      }
     }
 
@@ -286,78 +299,93 @@
 
     #region IRegressionModel Members
-    public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
       return GetEstimatedValuesHelper(dataset, rows);
     }
-    public GaussianProcessRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
       return new GaussianProcessRegressionSolution(this, new RegressionProblemData(problemData));
     }
-    IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
-      return CreateRegressionSolution(problemData);
-    }
     #endregion
 
 
     private IEnumerable<double> GetEstimatedValuesHelper(IDataset dataset, IEnumerable<int> rows) {
-      if (x == null) {
-        this.x = CalculateX(trainingDataset, allowedInputVariables, trainingRows, inputScaling);
-      }
-      int n = x.GetLength(0);
-
-      var newX = AlglibUtil.PrepareAndScaleInputMatrix(dataset, allowedInputVariables, rows, inputScaling);
-      int newN = newX.GetLength(0);
-
-      var Ks = new double[newN, n];
-      var mean = meanFunction.GetParameterizedMeanFunction(meanParameter, Enumerable.Range(0, newX.GetLength(1)));
-      var ms = Enumerable.Range(0, newX.GetLength(0))
-      .Select(r => mean.Mean(newX, r))
-      .ToArray();
-      var cov = covarianceFunction.GetParameterizedCovarianceFunction(covarianceParameter, Enumerable.Range(0, newX.GetLength(1)));
-      for (int i = 0; i < newN; i++) {
-        for (int j = 0; j < n; j++) {
-          Ks[i, j] = cov.CrossCovariance(x, newX, j, i);
-        }
-      }
-
-      return Enumerable.Range(0, newN)
-        .Select(i => ms[i] + Util.ScalarProd(Util.GetRow(Ks, i), alpha));
+      try {
+        if (x == null) {
+          x = GetData(trainingDataset, allowedInputVariables, trainingRows, inputScaling);
+        }
+        int n = x.GetLength(0);
+
+        double[,] newX = GetData(dataset, allowedInputVariables, rows, inputScaling);
+        int newN = newX.GetLength(0);
+
+        var Ks = new double[newN][];
+        var columns = Enumerable.Range(0, newX.GetLength(1)).ToArray();
+        var mean = meanFunction.GetParameterizedMeanFunction(meanParameter, columns);
+        var ms = Enumerable.Range(0, newX.GetLength(0))
+        .Select(r => mean.Mean(newX, r))
+        .ToArray();
+        var cov = covarianceFunction.GetParameterizedCovarianceFunction(covarianceParameter, columns);
+        for (int i = 0; i < newN; i++) {
+          Ks[i] = new double[n];
+          for (int j = 0; j < n; j++) {
+            Ks[i][j] = cov.CrossCovariance(x, newX, j, i);
+          }
+        }
+
+        return Enumerable.Range(0, newN)
+          .Select(i => ms[i] + Util.ScalarProd(Ks[i], alpha));
+      }
+      catch (alglib.alglibexception ae) {
+        // wrap exception so that calling code doesn't have to know about alglib implementation
+        throw new ArgumentException("There was a problem in the calculation of the Gaussian process model", ae);
+      }
     }
 
     public IEnumerable<double> GetEstimatedVariance(IDataset dataset, IEnumerable<int> rows) {
-      if (x == null) {
-        this.x = CalculateX(trainingDataset, allowedInputVariables, trainingRows, inputScaling);
-      }
-      int n = x.GetLength(0);
-
-      var newX = AlglibUtil.PrepareAndScaleInputMatrix(dataset, allowedInputVariables, rows, inputScaling);
-      int newN = newX.GetLength(0);
-
-      var kss = new double[newN];
-      double[,] sWKs = new double[n, newN];
-      var cov = covarianceFunction.GetParameterizedCovarianceFunction(covarianceParameter, Enumerable.Range(0, x.GetLength(1)));
-
-      if (l == null) {
-        l = CalculateL(x, cov, sqrSigmaNoise);
-      }
-
-      // for stddev 
-      for (int i = 0; i < newN; i++)
-        kss[i] = cov.Covariance(newX, i, i);
-
-      for (int i = 0; i < newN; i++) {
-        for (int j = 0; j < n; j++) {
-          sWKs[j, i] = cov.CrossCovariance(x, newX, j, i) / Math.Sqrt(sqrSigmaNoise);
-        }
-      }
-
-      // for stddev 
-      alglib.ablas.rmatrixlefttrsm(n, newN, l, 0, 0, false, false, 0, ref sWKs, 0, 0);
-
-      for (int i = 0; i < newN; i++) {
-        var sumV = Util.ScalarProd(Util.GetCol(sWKs, i), Util.GetCol(sWKs, i));
-        kss[i] -= sumV;
-        if (kss[i] < 0) kss[i] = 0;
-      }
-      return kss;
-    }
+      try {
+        if (x == null) {
+          x = GetData(trainingDataset, allowedInputVariables, trainingRows, inputScaling);
+        }
+        int n = x.GetLength(0);
+
+        var newX = GetData(dataset, allowedInputVariables, rows, inputScaling);
+        int newN = newX.GetLength(0);
+
+        var kss = new double[newN];
+        double[,] sWKs = new double[n, newN];
+        var columns = Enumerable.Range(0, newX.GetLength(1)).ToArray();
+        var cov = covarianceFunction.GetParameterizedCovarianceFunction(covarianceParameter, columns);
+
+        if (l == null) {
+          l = CalculateL(x, cov, sqrSigmaNoise);
+        }
+
+        // for stddev 
+        for (int i = 0; i < newN; i++)
+          kss[i] = cov.Covariance(newX, i, i);
+
+        for (int i = 0; i < newN; i++) {
+          for (int j = 0; j < n; j++) {
+            sWKs[j, i] = cov.CrossCovariance(x, newX, j, i) / Math.Sqrt(sqrSigmaNoise);
+          }
+        }
+
+        // for stddev 
+        alglib.ablas.rmatrixlefttrsm(n, newN, l, 0, 0, false, false, 0, ref sWKs, 0, 0);
+
+        for (int i = 0; i < newN; i++) {
+          var col = Util.GetCol(sWKs, i).ToArray();
+          var sumV = Util.ScalarProd(col, col);
+          kss[i] += sqrSigmaNoise; // kss is V(f), add noise variance of predictive distibution to get V(y)
+          kss[i] -= sumV;
+          if (kss[i] < 0) kss[i] = 0;
+        }
+        return kss;
+      }
+      catch (alglib.alglibexception ae) {
+        // wrap exception so that calling code doesn't have to know about alglib implementation
+        throw new ArgumentException("There was a problem in the calculation of the Gaussian process model", ae);
+      }
+    }
+
   }
 }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModelCreator.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModelCreator.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModelCreator.cs	(revision 14029)
@@ -38,4 +38,5 @@
     private const string NegativeLogLikelihoodParameterName = "NegativeLogLikelihood";
     private const string HyperparameterGradientsParameterName = "HyperparameterGradients";
+    protected const string ScaleInputValuesParameterName = "ScaleInputValues";
 
     #region Parameter Properties
@@ -60,5 +61,7 @@
       get { return (ILookupParameter<DoubleValue>)Parameters[NegativeLogLikelihoodParameterName]; }
     }
-
+    public ILookupParameter<BoolValue> ScaleInputValuesParameter {
+      get { return (ILookupParameter<BoolValue>)Parameters[ScaleInputValuesParameterName]; }
+    }
     #endregion
 
@@ -67,4 +70,5 @@
     protected IMeanFunction MeanFunction { get { return MeanFunctionParameter.ActualValue; } }
     protected ICovarianceFunction CovarianceFunction { get { return CovarianceFunctionParameter.ActualValue; } }
+    public bool ScaleInputValues { get { return ScaleInputValuesParameter.ActualValue.Value; } }
     #endregion
 
@@ -82,4 +86,18 @@
       Parameters.Add(new LookupParameter<RealVector>(HyperparameterGradientsParameterName, "The gradients of the hyperparameters for the produced Gaussian process model (necessary for hyperparameter optimization)"));
       Parameters.Add(new LookupParameter<DoubleValue>(NegativeLogLikelihoodParameterName, "The negative log-likelihood of the produced Gaussian process model given the data."));
+
+
+      Parameters.Add(new LookupParameter<BoolValue>(ScaleInputValuesParameterName,
+        "Determines if the input variable values are scaled to the range [0..1] for training."));
+      Parameters[ScaleInputValuesParameterName].Hidden = true;
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      if (!Parameters.ContainsKey(ScaleInputValuesParameterName)) {
+        Parameters.Add(new LookupParameter<BoolValue>(ScaleInputValuesParameterName,
+          "Determines if the input variable values are scaled to the range [0..1] for training."));
+        Parameters[ScaleInputValuesParameterName].Hidden = true;
+      }
     }
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegression.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegression.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegression.cs	(revision 14029)
@@ -25,4 +25,5 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Data;
 using HeuristicLab.Optimization;
 using HeuristicLab.Parameters;
@@ -48,4 +49,6 @@
 
     private const string ModelParameterName = "Model";
+    private const string CreateSolutionParameterName = "CreateSolution";
+
 
     #region parameter properties
@@ -55,4 +58,13 @@
     public IFixedValueParameter<GaussianProcessRegressionSolutionCreator> GaussianProcessSolutionCreatorParameter {
       get { return (IFixedValueParameter<GaussianProcessRegressionSolutionCreator>)Parameters[SolutionCreatorParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> CreateSolutionParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
+    }
+    #endregion
+    #region properties
+    public bool CreateSolution {
+      get { return CreateSolutionParameter.Value.Value; }
+      set { CreateSolutionParameter.Value.Value = value; }
     }
     #endregion
@@ -78,8 +90,11 @@
       Parameters.Add(new ConstrainedValueParameter<IGaussianProcessRegressionModelCreator>(ModelCreatorParameterName, "The operator to create the Gaussian process model.",
         new ItemSet<IGaussianProcessRegressionModelCreator>(modelCreators), defaultModelCreator));
-      // this parameter is not intended to be changed, 
+      // the solution creator cannot be changed
       Parameters.Add(new FixedValueParameter<GaussianProcessRegressionSolutionCreator>(SolutionCreatorParameterName, "The solution creator for the algorithm",
         new GaussianProcessRegressionSolutionCreator()));
       Parameters[SolutionCreatorParameterName].Hidden = true;
+      // TODO: it would be better to deactivate the solution creator when this parameter is changed
+      Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+      Parameters[CreateSolutionParameterName].Hidden = true;
 
       ParameterizedModelCreators();
@@ -91,4 +106,11 @@
     [StorableHook(HookType.AfterDeserialization)]
     private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
+      if (!Parameters.ContainsKey(CreateSolutionParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+        Parameters[CreateSolutionParameterName].Hidden = true;
+      }
+      #endregion
       RegisterEventHandlers();
     }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs	(revision 14029)
@@ -62,5 +62,5 @@
     public override IOperation Apply() {
       try {
-        var model = Create(ProblemData, Hyperparameter.ToArray(), MeanFunction, CovarianceFunction);
+        var model = Create(ProblemData, Hyperparameter.ToArray(), MeanFunction, CovarianceFunction, ScaleInputValues);
         ModelParameter.ActualValue = model;
         NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(model.NegativeLogLikelihood);
@@ -75,6 +75,6 @@
     }
 
-    public static IGaussianProcessModel Create(IRegressionProblemData problemData, double[] hyperparameter, IMeanFunction meanFunction, ICovarianceFunction covarianceFunction) {
-      return new GaussianProcessModel(problemData.Dataset, problemData.TargetVariable, problemData.AllowedInputVariables, problemData.TrainingIndices, hyperparameter, meanFunction, covarianceFunction);
+    public static IGaussianProcessModel Create(IRegressionProblemData problemData, double[] hyperparameter, IMeanFunction meanFunction, ICovarianceFunction covarianceFunction, bool scaleInputs = true) {
+      return new GaussianProcessModel(problemData.Dataset, problemData.TargetVariable, problemData.AllowedInputVariables, problemData.TrainingIndices, hyperparameter, meanFunction, covarianceFunction, scaleInputs);
     }
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs	(revision 14029)
@@ -40,4 +40,5 @@
     private const string TrainingRSquaredResultName = "Training R²";
     private const string TestRSquaredResultName = "Test R²";
+    private const string CreateSolutionParameterName = "CreateSolution";
 
     #region Parameter Properties
@@ -54,4 +55,7 @@
       get { return (ILookupParameter<ResultCollection>)Parameters[ResultsParameterName]; }
     }
+    public ILookupParameter<BoolValue> CreateSolutionParameter {
+      get { return (ILookupParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
+    }
     #endregion
 
@@ -64,8 +68,20 @@
       Parameters.Add(new LookupParameter<IRegressionProblemData>(ProblemDataParameterName, "The regression problem data for the Gaussian process solution."));
       Parameters.Add(new LookupParameter<IGaussianProcessModel>(ModelParameterName, "The Gaussian process regression model to use for the solution."));
+      Parameters.Add(new LookupParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run"));
+
       // in & out
       Parameters.Add(new LookupParameter<ResultCollection>(ResultsParameterName, "The result collection of the algorithm."));
       // out
       Parameters.Add(new LookupParameter<IGaussianProcessSolution>(SolutionParameterName, "The produced Gaussian process solution."));
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
+      if (!Parameters.ContainsKey(CreateSolutionParameterName)) {
+        Parameters.Add(new LookupParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run"));
+      }
+      #endregion
     }
 
@@ -75,5 +91,5 @@
 
     public override IOperation Apply() {
-      if (ModelParameter.ActualValue != null) {
+      if (ModelParameter.ActualValue != null && CreateSolutionParameter.ActualValue.Value == true) {
         var m = (IGaussianProcessModel)ModelParameter.ActualValue.Clone();
         m.FixParameters();
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/ICovarianceFunction.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/ICovarianceFunction.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/ICovarianceFunction.cs	(revision 14029)
@@ -20,7 +20,5 @@
 #endregion
 
-using System;
 using System.Collections.Generic;
-using System.Linq.Expressions;
 using HeuristicLab.Core;
 
@@ -29,5 +27,5 @@
   public delegate double CovarianceFunctionDelegate(double[,] x, int i, int j);
   public delegate double CrossCovarianceFunctionDelegate(double[,] x, double[,] xt, int i, int j);
-  public delegate IEnumerable<double> CovarianceGradientFunctionDelegate(double[,] x, int i, int j);
+  public delegate IList<double> CovarianceGradientFunctionDelegate(double[,] x, int i, int j);
 
   public class ParameterizedCovarianceFunction {
@@ -40,5 +38,5 @@
     int GetNumberOfParameters(int numberOfVariables);
     void SetParameter(double[] p);
-    ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices);
+    ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices);
   }
 }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/IMeanFunction.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/IMeanFunction.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/IMeanFunction.cs	(revision 14029)
@@ -20,6 +20,4 @@
 #endregion
 
-using System;
-using System.Collections.Generic;
 using HeuristicLab.Core;
 
@@ -36,5 +34,5 @@
     int GetNumberOfParameters(int numberOfVariables);
     void SetParameter(double[] p);
-    ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, IEnumerable<int> columnIndices);
+    ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, int[] columnIndices);
   }
 }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanConst.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanConst.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanConst.cs	(revision 14029)
@@ -76,5 +76,5 @@
     }
 
-    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, int[] columnIndices) {
       double c;
       GetParameters(p, out c);
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanLinear.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanLinear.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanLinear.cs	(revision 14029)
@@ -21,5 +21,4 @@
 
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using HeuristicLab.Common;
@@ -70,7 +69,7 @@
     }
 
-    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, int[] columnIndices) {
       double[] weights;
-      int[] columns = columnIndices.ToArray();
+      int[] columns = columnIndices;
       GetParameter(p, out weights);
       var mf = new ParameterizedMeanFunction();
@@ -78,5 +77,5 @@
         // sanity check
         if (weights.Length != columns.Length) throw new ArgumentException("The number of rparameters must match the number of variables for the linear mean function.");
-        return Util.ScalarProd(weights, Util.GetRow(x, i, columns));
+        return Util.ScalarProd(weights, Util.GetRow(x, i, columns).ToArray());
       };
       mf.Gradient = (x, i, k) => {
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanModel.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanModel.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanModel.cs	(revision 14029)
@@ -0,0 +1,98 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "MeanModel", Description = "A mean function for Gaussian processes that uses a regression solution created with a different algorithm to calculate the mean.")]
+  // essentially an adaptor which maps from IMeanFunction to IRegressionSolution
+  public sealed class MeanModel : ParameterizedNamedItem, IMeanFunction {
+    public IValueParameter<IRegressionSolution> RegressionSolutionParameter {
+      get { return (IValueParameter<IRegressionSolution>)Parameters["RegressionSolution"]; }
+    }
+    public IRegressionSolution RegressionSolution {
+      get { return RegressionSolutionParameter.Value; }
+      set { RegressionSolutionParameter.Value = value; }
+    }
+
+    [StorableConstructor]
+    private MeanModel(bool deserializing) : base(deserializing) { }
+    private MeanModel(MeanModel original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public MeanModel()
+      : base("MeanModel", "A mean function for Gaussian processes that uses a regression solution created with a different algorithm to calculate the mean.") {
+      Parameters.Add(new ValueParameter<IRegressionSolution>("RegressionSolution", "The solution containing the model that should be used for the mean prediction."));
+    }
+
+    public MeanModel(IRegressionSolution solution)
+      : this() {
+      // here we cannot check if the model is actually compatible (uses only input variables that are available)
+      // we only assume that the list of allowed inputs in the regression solution is the same as the list of allowed
+      // inputs in the Gaussian process.
+      // later we might get an error or bad behaviour when the mean function is evaluated
+      RegressionSolution = solution;
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanModel(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return 0; // no support for hyperparameters for regression models yet
+    }
+
+    public void SetParameter(double[] p) {
+      if (p.Length > 0) throw new ArgumentException("No parameters allowed for model-based mean function.", "p");
+    }
+
+    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, int[] columnIndices) {
+      if (p.Length > 0) throw new ArgumentException("No parameters allowed for model-based mean function.", "p");
+      var solution = RegressionSolution;
+      var variableNames = solution.ProblemData.AllowedInputVariables.ToArray();
+      if (variableNames.Length != columnIndices.Length)
+        throw new ArgumentException("The number of input variables does not match in MeanModel");
+      var variableValues = variableNames.Select(_ => new List<double>() { 0.0 }).ToArray(); // or of zeros
+      // uses modifyable dataset to pass values to the model
+      var ds = new ModifiableDataset(variableNames, variableValues);
+      var mf = new ParameterizedMeanFunction();
+      var model = solution.Model; // effort for parameter access only once
+      mf.Mean = (x, i) => {
+        ds.ReplaceRow(0, Util.GetRow(x, i, columnIndices).OfType<object>());
+        return model.GetEstimatedValues(ds, 0.ToEnumerable()).Single(); // evaluate the model on the specified row only
+      };
+      mf.Gradient = (x, i, k) => {
+        if (k > 0)
+          throw new ArgumentException();
+        return 0.0;
+      };
+      return mf;
+    }
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanProduct.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanProduct.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanProduct.cs	(revision 14029)
@@ -73,5 +73,5 @@
 
 
-    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, int[] columnIndices) {
       var factorMf = new List<ParameterizedMeanFunction>();
       int totalNumberOfParameters = GetNumberOfParameters(numberOfVariables);
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanSum.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanSum.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanSum.cs	(revision 14029)
@@ -68,5 +68,5 @@
     }
 
-    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, int[] columnIndices) {
       var termMf = new List<ParameterizedMeanFunction>();
       int totalNumberOfParameters = GetNumberOfParameters(numberOfVariables);
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanZero.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanZero.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanZero.cs	(revision 14029)
@@ -20,6 +20,4 @@
 #endregion
 using System;
-using System.Collections.Generic;
-using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -50,5 +48,5 @@
     }
 
-    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, IEnumerable<int> columnIndices) {
+    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, int[] columnIndices) {
       if (p.Length > 0) throw new ArgumentException("No parameters allowed for zero mean function.", "p");
       var mf = new ParameterizedMeanFunction();
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/Util.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/Util.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/Util.cs	(revision 14029)
@@ -23,11 +23,13 @@
 using System.Collections.Generic;
 using System.Linq;
-using HeuristicLab.Core;
-using HeuristicLab.Data;
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
   internal static class Util {
-    public static double ScalarProd(IEnumerable<double> v, IEnumerable<double> u) {
-      return v.Zip(u, (vi, ui) => vi * ui).Sum();
+    public static double ScalarProd(double[] v, double[] u) {
+      if (v.Length != u.Length) throw new InvalidOperationException();
+      double prod = 0.0;
+      for (int i = 0; i < v.Length; i++)
+        prod += v[i] * u[i];
+      return prod;
     }
 
@@ -41,12 +43,13 @@
     }
 
-    public static double SqrDist(double[,] x, int i, int j, double scale = 1.0, IEnumerable<int> columnIndices = null) {
-      return SqrDist(x, i, x, j, scale, columnIndices);
+    public static double SqrDist(double[,] x, int i, int j, int[] columnIndices, double scale = 1.0) {
+      return SqrDist(x, i, x, j, columnIndices, scale);
     }
 
-    public static double SqrDist(double[,] x, int i, double[,] xt, int j, double scale = 1.0, IEnumerable<int> columnIndices = null) {
+    public static double SqrDist(double[,] x, int i, double[,] xt, int j, int[] columnIndices, double scale = 1.0) {
       double ss = 0.0;
-      if (columnIndices == null) columnIndices = Enumerable.Range(0, x.GetLength(1));
-      foreach (int columnIndex in columnIndices) {
+      if (columnIndices == null) columnIndices = Enumerable.Range(0, x.GetLength(1)).ToArray();
+      for (int c = 0; c < columnIndices.Length; c++) {
+        var columnIndex = columnIndices[c];
         double d = x[i, columnIndex] - xt[j, columnIndex];
         ss += d * d;
@@ -55,13 +58,13 @@
     }
 
-    public static double SqrDist(double[,] x, int i, int j, double[] scale, IEnumerable<int> columnIndices = null) {
+    public static double SqrDist(double[,] x, int i, int j, double[] scale, int[] columnIndices) {
       return SqrDist(x, i, x, j, scale, columnIndices);
     }
 
-    public static double SqrDist(double[,] x, int i, double[,] xt, int j, double[] scale, IEnumerable<int> columnIndices = null) {
+    public static double SqrDist(double[,] x, int i, double[,] xt, int j, double[] scale, int[] columnIndices) {
       double ss = 0.0;
-      if (columnIndices == null) columnIndices = Enumerable.Range(0, x.GetLength(1));
       int scaleIndex = 0;
-      foreach (int columnIndex in columnIndices) {
+      for (int c = 0; c < columnIndices.Length; c++) {
+        var columnIndex = columnIndices[c];
         double d = x[i, columnIndex] - xt[j, columnIndex];
         ss += d * d * scale[scaleIndex] * scale[scaleIndex];
@@ -73,27 +76,26 @@
       return ss;
     }
-    public static double ScalarProd(double[,] x, int i, int j, double scale = 1.0, IEnumerable<int> columnIndices = null) {
-      return ScalarProd(x, i, x, j, scale, columnIndices);
+    public static double ScalarProd(double[,] x, int i, int j, int[] columnIndices, double scale = 1.0) {
+      return ScalarProd(x, i, x, j, columnIndices, scale);
     }
 
-    public static double ScalarProd(double[,] x, int i, double[,] xt, int j, double scale = 1.0, IEnumerable<int> columnIndices = null) {
+    public static double ScalarProd(double[,] x, int i, double[,] xt, int j, int[] columnIndices, double scale = 1.0) {
       double sum = 0.0;
-      if (columnIndices == null) columnIndices = Enumerable.Range(0, x.GetLength(1));
-      foreach (int columnIndex in columnIndices) {
+      for (int c = 0; c < columnIndices.Length; c++) {
+        var columnIndex = columnIndices[c];
         sum += x[i, columnIndex] * xt[j, columnIndex];
       }
       return scale * scale * sum;
     }
-    public static double ScalarProd(double[,] x, int i, int j, double[] scale, IEnumerable<int> columnIndices = null) {
+    public static double ScalarProd(double[,] x, int i, int j, double[] scale, int[] columnIndices) {
       return ScalarProd(x, i, x, j, scale, columnIndices);
     }
 
-    public static double ScalarProd(double[,] x, int i, double[,] xt, int j, double[] scale, IEnumerable<int> columnIndices = null) {
+    public static double ScalarProd(double[,] x, int i, double[,] xt, int j, double[] scale, int[] columnIndices) {
       double sum = 0.0;
-      if (columnIndices == null) columnIndices = Enumerable.Range(0, x.GetLength(1));
       int scaleIndex = 0;
-      foreach (int columnIndex in columnIndices) {
+      for (int c = 0; c < columnIndices.Length; c++, scaleIndex++) {
+        var columnIndex = columnIndices[c];
         sum += x[i, columnIndex] * scale[scaleIndex] * xt[j, columnIndex] * scale[scaleIndex];
-        scaleIndex++;
       }
       // must be at the end of scale after iterating over columnIndices
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithm.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithm.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithm.cs	(revision 14029)
@@ -35,5 +35,5 @@
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
-  [Item("Gradient Boosted Trees", "Gradient boosted trees algorithm. Friedman, J. \"Greedy Function Approximation: A Gradient Boosting Machine\", IMS 1999 Reitz Lecture.")]
+  [Item("Gradient Boosted Trees (GBT)", "Gradient boosted trees algorithm. Specific implementation of gradient boosting for regression trees. Friedman, J. \"Greedy Function Approximation: A Gradient Boosting Machine\", IMS 1999 Reitz Lecture.")]
   [StorableClass]
   [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 125)]
@@ -255,20 +255,19 @@
       // produce solution 
       if (CreateSolution) {
-        var surrogateModel = new GradientBoostedTreesModelSurrogate(problemData, (uint)Seed, lossFunction,
-          Iterations, MaxSize, R, M, Nu, state.GetModel());
+        var model = state.GetModel();
 
         // for logistic regression we produce a classification solution
         if (lossFunction is LogisticRegressionLoss) {
-          var model = new DiscriminantFunctionClassificationModel(surrogateModel,
+          var classificationModel = new DiscriminantFunctionClassificationModel(model,
             new AccuracyMaximizationThresholdCalculator());
           var classificationProblemData = new ClassificationProblemData(problemData.Dataset,
             problemData.AllowedInputVariables, problemData.TargetVariable, problemData.Transformations);
-          model.RecalculateModelParameters(classificationProblemData, classificationProblemData.TrainingIndices);
-
-          var classificationSolution = new DiscriminantFunctionClassificationSolution(model, classificationProblemData);
+          classificationModel.RecalculateModelParameters(classificationProblemData, classificationProblemData.TrainingIndices);
+
+          var classificationSolution = new DiscriminantFunctionClassificationSolution(classificationModel, classificationProblemData);
           Results.Add(new Result("Solution", classificationSolution));
         } else {
           // otherwise we produce a regression solution
-          Results.Add(new Result("Solution", new RegressionSolution(surrogateModel, problemData)));
+          Results.Add(new Result("Solution", new RegressionSolution(model, problemData)));
         }
       }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithmStatic.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithmStatic.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithmStatic.cs	(revision 14029)
@@ -52,4 +52,5 @@
       internal RegressionTreeBuilder treeBuilder { get; private set; }
 
+      private readonly uint randSeed;
       private MersenneTwister random { get; set; }
 
@@ -71,4 +72,5 @@
         this.m = m;
 
+        this.randSeed = randSeed;
         random = new MersenneTwister(randSeed);
         this.problemData = problemData;
@@ -94,10 +96,16 @@
         weights = new List<double>();
         // add constant model
-        models.Add(new ConstantRegressionModel(f0));
+        models.Add(new ConstantModel(f0, problemData.TargetVariable));
         weights.Add(1.0);
       }
 
       public IRegressionModel GetModel() {
-        return new GradientBoostedTreesModel(models, weights);
+#pragma warning disable 618
+        var model = new GradientBoostedTreesModel(models, weights);
+#pragma warning restore 618
+        // we don't know the number of iterations here but the number of weights is equal 
+        // to the number of iterations + 1 (for the constant model)
+        // wrap the actual model in a surrogate that enables persistence and lazy recalculation of the model if necessary
+        return new GradientBoostedTreesModelSurrogate(problemData, randSeed, lossFunction, weights.Count - 1, maxSize, r, m, nu, model);
       }
       public IEnumerable<KeyValuePair<string, double>> GetVariableRelevance() {
@@ -122,5 +130,5 @@
 
     // simple interface
-    public static IRegressionSolution TrainGbm(IRegressionProblemData problemData, ILossFunction lossFunction, int maxSize, double nu, double r, double m, int maxIterations, uint randSeed = 31415) {
+    public static GradientBoostedTreesSolution TrainGbm(IRegressionProblemData problemData, ILossFunction lossFunction, int maxSize, double nu, double r, double m, int maxIterations, uint randSeed = 31415) {
       Contract.Assert(r > 0);
       Contract.Assert(r <= 1.0);
@@ -135,5 +143,5 @@
 
       var model = state.GetModel();
-      return new RegressionSolution(model, (IRegressionProblemData)problemData.Clone());
+      return new GradientBoostedTreesSolution(model, (IRegressionProblemData)problemData.Clone());
     }
 
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesModel.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesModel.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesModel.cs	(revision 14029)
@@ -33,5 +33,5 @@
   [Item("Gradient boosted tree model", "")]
   // this is essentially a collection of weighted regression models
-  public sealed class GradientBoostedTreesModel : NamedItem, IRegressionModel {
+  public sealed class GradientBoostedTreesModel : RegressionModel, IGradientBoostedTreesModel {
     // BackwardsCompatibility3.4 for allowing deserialization & serialization of old models
     #region Backwards compatible code, remove with 3.5
@@ -58,4 +58,8 @@
     #endregion
 
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return models.SelectMany(x => x.VariablesUsedForPrediction).Distinct().OrderBy(x => x); }
+    }
+
     private readonly IList<IRegressionModel> models;
     public IEnumerable<IRegressionModel> Models { get { return models; } }
@@ -76,6 +80,7 @@
       this.isCompatibilityLoaded = original.isCompatibilityLoaded;
     }
-    public GradientBoostedTreesModel(IEnumerable<IRegressionModel> models, IEnumerable<double> weights)
-      : base("Gradient boosted tree model", string.Empty) {
+    [Obsolete("The constructor of GBTModel should not be used directly anymore (use GBTModelSurrogate instead)")]
+    internal GradientBoostedTreesModel(IEnumerable<IRegressionModel> models, IEnumerable<double> weights)
+      : base(string.Empty, "Gradient boosted tree model", string.Empty) {
       this.models = new List<IRegressionModel>(models);
       this.weights = new List<double>(weights);
@@ -88,5 +93,5 @@
     }
 
-    public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
       // allocate target array go over all models and add up weighted estimation for each row
       if (!rows.Any()) return Enumerable.Empty<double>(); // return immediately if rows is empty. This prevents multiple iteration over lazy rows enumerable.
@@ -104,7 +109,8 @@
     }
 
-    public IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
       return new RegressionSolution(this, (IRegressionProblemData)problemData.Clone());
     }
+
   }
 }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesModelSurrogate.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesModelSurrogate.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesModelSurrogate.cs	(revision 14029)
@@ -21,5 +21,4 @@
 #endregion
 
-using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -27,5 +26,4 @@
 using HeuristicLab.Core;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using HeuristicLab.PluginInfrastructure;
 using HeuristicLab.Problems.DataAnalysis;
 
@@ -36,7 +34,7 @@
   // recalculate the actual GBT model on demand
   [Item("Gradient boosted tree model", "")]
-  public sealed class GradientBoostedTreesModelSurrogate : NamedItem, IRegressionModel {
+  public sealed class GradientBoostedTreesModelSurrogate : RegressionModel, IGradientBoostedTreesModel {
     // don't store the actual model!
-    private IRegressionModel actualModel; // the actual model is only recalculated when necessary
+    private IGradientBoostedTreesModel actualModel; // the actual model is only recalculated when necessary
 
     [Storable]
@@ -58,4 +56,8 @@
 
 
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return actualModel.Models.SelectMany(x => x.VariablesUsedForPrediction).Distinct().OrderBy(x => x); }
+    }
+
     [StorableConstructor]
     private GradientBoostedTreesModelSurrogate(bool deserializing) : base(deserializing) { }
@@ -76,6 +78,7 @@
 
     // create only the surrogate model without an actual model
-    public GradientBoostedTreesModelSurrogate(IRegressionProblemData trainingProblemData, uint seed, ILossFunction lossFunction, int iterations, int maxSize, double r, double m, double nu)
-      : base("Gradient boosted tree model", string.Empty) {
+    public GradientBoostedTreesModelSurrogate(IRegressionProblemData trainingProblemData, uint seed,
+      ILossFunction lossFunction, int iterations, int maxSize, double r, double m, double nu)
+      : base(trainingProblemData.TargetVariable, "Gradient boosted tree model", string.Empty) {
       this.trainingProblemData = trainingProblemData;
       this.seed = seed;
@@ -89,5 +92,7 @@
 
     // wrap an actual model in a surrograte
-    public GradientBoostedTreesModelSurrogate(IRegressionProblemData trainingProblemData, uint seed, ILossFunction lossFunction, int iterations, int maxSize, double r, double m, double nu, IRegressionModel model)
+    public GradientBoostedTreesModelSurrogate(IRegressionProblemData trainingProblemData, uint seed,
+      ILossFunction lossFunction, int iterations, int maxSize, double r, double m, double nu,
+      IGradientBoostedTreesModel model)
       : this(trainingProblemData, seed, lossFunction, iterations, maxSize, r, m, nu) {
       this.actualModel = model;
@@ -99,16 +104,29 @@
 
     // forward message to actual model (recalculate model first if necessary)
-    public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
       if (actualModel == null) actualModel = RecalculateModel();
       return actualModel.GetEstimatedValues(dataset, rows);
     }
 
-    public IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
       return new RegressionSolution(this, (IRegressionProblemData)problemData.Clone());
     }
 
+    private IGradientBoostedTreesModel RecalculateModel() {
+      return GradientBoostedTreesAlgorithmStatic.TrainGbm(trainingProblemData, lossFunction, maxSize, nu, r, m, iterations, seed).Model;
+    }
 
-    private IRegressionModel RecalculateModel() {
-      return GradientBoostedTreesAlgorithmStatic.TrainGbm(trainingProblemData, lossFunction, maxSize, nu, r, m, iterations, seed).Model;
+    public IEnumerable<IRegressionModel> Models {
+      get {
+        if (actualModel == null) actualModel = RecalculateModel();
+        return actualModel.Models;
+      }
+    }
+
+    public IEnumerable<double> Weights {
+      get {
+        if (actualModel == null) actualModel = RecalculateModel();
+        return actualModel.Weights;
+      }
     }
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesSolution.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesSolution.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesSolution.cs	(revision 14029)
@@ -0,0 +1,51 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  public sealed class GradientBoostedTreesSolution : RegressionSolution {
+    public new IGradientBoostedTreesModel Model {
+      get { return (IGradientBoostedTreesModel)base.Model; }
+    }
+
+
+    [StorableConstructor]
+    private GradientBoostedTreesSolution(bool deserializing)
+      : base(deserializing) {
+    }
+    private GradientBoostedTreesSolution(GradientBoostedTreesSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public GradientBoostedTreesSolution(IRegressionModel model, IRegressionProblemData problemData)
+      : base(model, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GradientBoostedTreesSolution(this, cloner);
+    }
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/IGradientBoostedTreesModel.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/IGradientBoostedTreesModel.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/IGradientBoostedTreesModel.cs	(revision 14029)
@@ -0,0 +1,36 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public interface IGradientBoostedTreesModel : IRegressionModel {
+    IEnumerable<IRegressionModel> Models { get; }
+    IEnumerable<double> Weights { get; }
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/QuantileRegressionLoss.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/QuantileRegressionLoss.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/QuantileRegressionLoss.cs	(revision 14029)
@@ -0,0 +1,118 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  // loss function for quantile regression
+  // Generalized Boosted Models - A Guide To The gbm Package, Greg Ridgeway, August 2007, page 11 
+  [StorableClass]
+  [Item("QuantileRegressionloss", "Loss function for quantile regression")]
+  public sealed class QuantileRegressionLoss : ParameterizedNamedItem, ILossFunction {
+    public IFixedValueParameter<PercentValue> AlphaParameter {
+      get { return (IFixedValueParameter<PercentValue>)Parameters["Alpha"]; }
+    }
+
+    public double Alpha {
+      get { return AlphaParameter.Value.Value; }
+      set {
+        if (value <= 0.0 || value >= 1.0) throw new ArgumentException("Valid values for alpha: 0 < alpha < 1");
+        AlphaParameter.Value.Value = value;
+      }
+    }
+
+    public QuantileRegressionLoss()
+      : base("QuantileRegressionLoss", "Loss function for quantile regression") {
+      Parameters.Add(new FixedValueParameter<PercentValue>("Alpha", new PercentValue(0.9)));
+    }
+
+    public double GetLoss(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+      var alpha = Alpha;
+      double leftSum = 0;
+      double rightsum = 0;
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        double res = targetEnum.Current - predEnum.Current;
+        if (res > 0) leftSum += res;
+        else rightsum += -res;
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have differing lengths");
+
+      return alpha * leftSum + (1 - alpha) * rightsum;
+    }
+
+    public IEnumerable<double> GetLossGradient(IEnumerable<double> target, IEnumerable<double> pred) {
+      var targetEnum = target.GetEnumerator();
+      var predEnum = pred.GetEnumerator();
+      var alpha = AlphaParameter.Value.Value;
+
+      while (targetEnum.MoveNext() & predEnum.MoveNext()) {
+        var res = targetEnum.Current - predEnum.Current;
+        if (res > 0) yield return alpha;
+        else if (res < 0) yield return -(1.0 - alpha);
+        else yield return 0.0;
+      }
+      if (targetEnum.MoveNext() | predEnum.MoveNext())
+        throw new ArgumentException("target and pred have differing lengths");
+    }
+
+    // targetArr and predArr are not changed by LineSearch
+    public double LineSearch(double[] targetArr, double[] predArr, int[] idx, int startIdx, int endIdx) {
+      if (targetArr.Length != predArr.Length)
+        throw new ArgumentException("target and pred have differing lengths");
+
+      // Quantile() is allocating an array anyway
+      // It would be possible to pre-allocated an array for the residuals if Quantile() would allow specification of a sub-range
+      int nRows = endIdx - startIdx + 1;
+      var res = new double[nRows];
+      for (int i = startIdx; i <= endIdx; i++) {
+        var row = idx[i];
+        res[i - startIdx] = targetArr[row] - predArr[row];
+      }
+      return res.Quantile(Alpha);
+    }
+
+    #region item implementation
+    [StorableConstructor]
+    private QuantileRegressionLoss(bool deserializing) : base(deserializing) { }
+
+    private QuantileRegressionLoss(QuantileRegressionLoss original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new QuantileRegressionLoss(this, cloner);
+    }
+    #endregion
+
+    public override bool CanChangeName { get { return false; } }
+    public override bool CanChangeDescription { get { return false; } }
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeBuilder.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeBuilder.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeBuilder.cs	(revision 14029)
@@ -119,23 +119,4 @@
     }
 
-    // simple API produces a single regression tree optimizing sum of squared errors
-    // this can be used if only a simple regression tree should be produced
-    // for a set of trees use the method CreateRegressionTreeForGradientBoosting below
-    //
-    // r and m work in the same way as for alglib random forest
-    // r is fraction of rows to use for training
-    // m is fraction of variables to use for training
-    public IRegressionModel CreateRegressionTree(int maxSize, double r = 0.5, double m = 0.5) {
-      // subtract mean of y first
-      var yAvg = y.Average();
-      for (int i = 0; i < y.Length; i++) y[i] -= yAvg;
-
-      var seLoss = new SquaredErrorLoss();
-
-      var model = CreateRegressionTreeForGradientBoosting(y, curPred, maxSize, problemData.TrainingIndices.ToArray(), seLoss, r, m);
-
-      return new GradientBoostedTreesModel(new[] { new ConstantRegressionModel(yAvg), model }, new[] { 1.0, 1.0 });
-    }
-
     // specific interface that allows to specify the target labels and the training rows which is necessary when for gradient boosted trees
     public IRegressionModel CreateRegressionTreeForGradientBoosting(double[] y, double[] curPred, int maxSize, int[] idx, ILossFunction lossFunction, double r = 0.5, double m = 0.5) {
@@ -156,5 +137,5 @@
       int nRows = idx.Count();
 
-      // shuffle variable idx
+      // shuffle variable names
       HeuristicLab.Random.ListExtensions.ShuffleInPlace(allowedVariables, random);
 
@@ -195,9 +176,9 @@
       CreateRegressionTreeFromQueue(maxSize, lossFunction);
 
-      return new RegressionTreeModel(tree.ToArray());
-    }
-
-
-    // processes potential splits from the queue as long as splits are left and the maximum size of the tree is not reached
+      return new RegressionTreeModel(tree.ToArray(), problemData.TargetVariable);
+    }
+
+
+    // processes potential splits from the queue as long as splits are remaining and the maximum size of the tree is not reached
     private void CreateRegressionTreeFromQueue(int maxNodes, ILossFunction lossFunction) {
       while (queue.Any() && curTreeNodeIdx + 1 < maxNodes) { // two nodes are created in each loop
@@ -223,5 +204,5 @@
 
         // overwrite existing leaf node with an internal node
-        tree[f.ParentNodeIdx] = new RegressionTreeModel.TreeNode(f.SplittingVariable, f.SplittingThreshold, leftTreeIdx, rightTreeIdx);
+        tree[f.ParentNodeIdx] = new RegressionTreeModel.TreeNode(f.SplittingVariable, f.SplittingThreshold, leftTreeIdx, rightTreeIdx, weightLeft: (splitIdx - startIdx + 1) / (double)(endIdx - startIdx + 1));
       }
     }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeModel.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeModel.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeModel.cs	(revision 14029)
@@ -23,4 +23,5 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
 using System.Globalization;
 using System.Linq;
@@ -33,5 +34,8 @@
   [StorableClass]
   [Item("RegressionTreeModel", "Represents a decision tree for regression.")]
-  public sealed class RegressionTreeModel : NamedItem, IRegressionModel {
+  public sealed class RegressionTreeModel : RegressionModel {
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return tree.Select(t => t.VarName).Where(v => v != TreeNode.NO_VARIABLE); }
+    }
 
     // trees are represented as a flat array    
@@ -39,5 +43,5 @@
       public readonly static string NO_VARIABLE = null;
 
-      public TreeNode(string varName, double val, int leftIdx = -1, int rightIdx = -1)
+      public TreeNode(string varName, double val, int leftIdx = -1, int rightIdx = -1, double weightLeft = -1.0)
         : this() {
         VarName = varName;
@@ -45,12 +49,13 @@
         LeftIdx = leftIdx;
         RightIdx = rightIdx;
-      }
-
-      public string VarName { get; private set; } // name of the variable for splitting or NO_VARIABLE if terminal node
-      public double Val { get; private set; } // threshold
-      public int LeftIdx { get; private set; }
-      public int RightIdx { get; private set; }
-
-      internal IList<double> Data { get; set; } // only necessary to improve efficiency of evaluation
+        WeightLeft = weightLeft;
+      }
+
+      public string VarName { get; internal set; } // name of the variable for splitting or NO_VARIABLE if terminal node
+      public double Val { get; internal set; } // threshold
+      public int LeftIdx { get; internal set; }
+      public int RightIdx { get; internal set; }
+      public double WeightLeft { get; internal set; } // for partial dependence plots (value in range [0..1] describes the fraction of training samples for the left sub-tree
+
 
       // necessary because the default implementation of GetHashCode for structs in .NET would only return the hashcode of val here
@@ -65,4 +70,5 @@
             LeftIdx.Equals(other.LeftIdx) &&
             RightIdx.Equals(other.RightIdx) &&
+            WeightLeft.Equals(other.WeightLeft) &&
             EqualStrings(VarName, other.VarName);
         } else {
@@ -80,11 +86,66 @@
     private TreeNode[] tree;
 
-    [Storable]
+    #region old storable format
+    // remove with HL 3.4
+    [Storable(AllowOneWay = true)]
     // to prevent storing the references to data caches in nodes
-    // seemingly it is bad (performance-wise) to persist tuples (tuples are used as keys in a dictionary) TODO
+    // seemingly, it is bad (performance-wise) to persist tuples (tuples are used as keys in a dictionary)
     private Tuple<string, double, int, int>[] SerializedTree {
-      get { return tree.Select(t => Tuple.Create(t.VarName, t.Val, t.LeftIdx, t.RightIdx)).ToArray(); }
-      set { this.tree = value.Select(t => new TreeNode(t.Item1, t.Item2, t.Item3, t.Item4)).ToArray(); }
-    }
+      // get { return tree.Select(t => Tuple.Create(t.VarName, t.Val, t.LeftIdx, t.RightIdx)).ToArray(); }
+      set { this.tree = value.Select(t => new TreeNode(t.Item1, t.Item2, t.Item3, t.Item4, -1.0)).ToArray(); } // use a weight of -1.0 to indicate that partial dependence cannot be calculated for old models 
+    }
+    #endregion
+    #region new storable format
+    [Storable]
+    private string[] SerializedTreeVarNames {
+      get { return tree.Select(t => t.VarName).ToArray(); }
+      set {
+        if (tree == null) tree = new TreeNode[value.Length];
+        for (int i = 0; i < value.Length; i++) {
+          tree[i].VarName = value[i];
+        }
+      }
+    }
+    [Storable]
+    private double[] SerializedTreeValues {
+      get { return tree.Select(t => t.Val).ToArray(); }
+      set {
+        if (tree == null) tree = new TreeNode[value.Length];
+        for (int i = 0; i < value.Length; i++) {
+          tree[i].Val = value[i];
+        }
+      }
+    }
+    [Storable]
+    private int[] SerializedTreeLeftIdx {
+      get { return tree.Select(t => t.LeftIdx).ToArray(); }
+      set {
+        if (tree == null) tree = new TreeNode[value.Length];
+        for (int i = 0; i < value.Length; i++) {
+          tree[i].LeftIdx = value[i];
+        }
+      }
+    }
+    [Storable]
+    private int[] SerializedTreeRightIdx {
+      get { return tree.Select(t => t.RightIdx).ToArray(); }
+      set {
+        if (tree == null) tree = new TreeNode[value.Length];
+        for (int i = 0; i < value.Length; i++) {
+          tree[i].RightIdx = value[i];
+        }
+      }
+    }
+    [Storable]
+    private double[] SerializedTreeWeightLeft {
+      get { return tree.Select(t => t.WeightLeft).ToArray(); }
+      set {
+        if (tree == null) tree = new TreeNode[value.Length];
+        for (int i = 0; i < value.Length; i++) {
+          tree[i].WeightLeft = value[i];
+        }
+      }
+    }
+    #endregion
 
     [StorableConstructor]
@@ -99,16 +160,20 @@
     }
 
-    internal RegressionTreeModel(TreeNode[] tree)
-      : base("RegressionTreeModel", "Represents a decision tree for regression.") {
+    internal RegressionTreeModel(TreeNode[] tree, string targetVariable)
+      : base(targetVariable, "RegressionTreeModel", "Represents a decision tree for regression.") {
       this.tree = tree;
     }
 
-    private static double GetPredictionForRow(TreeNode[] t, int nodeIdx, int row) {
+    private static double GetPredictionForRow(TreeNode[] t, ReadOnlyCollection<double>[] columnCache, int nodeIdx, int row) {
       while (nodeIdx != -1) {
         var node = t[nodeIdx];
         if (node.VarName == TreeNode.NO_VARIABLE)
           return node.Val;
-
-        if (node.Data[row] <= node.Val)
+        if (columnCache[nodeIdx] == null || double.IsNaN(columnCache[nodeIdx][row])) {
+          if (node.WeightLeft.IsAlmost(-1.0)) throw new InvalidOperationException("Cannot calculate partial dependence for trees loaded from older versions of HeuristicLab.");
+          // weighted average for partial dependence plot (recursive here because we need to calculate both sub-trees)
+          return node.WeightLeft * GetPredictionForRow(t, columnCache, node.LeftIdx, row) +
+                 (1.0 - node.WeightLeft) * GetPredictionForRow(t, columnCache, node.RightIdx, row);
+        } else if (columnCache[nodeIdx][row] <= node.Val)
           nodeIdx = node.LeftIdx;
         else
@@ -122,15 +187,19 @@
     }
 
-    public IEnumerable<double> GetEstimatedValues(IDataset ds, IEnumerable<int> rows) {
+    public override IEnumerable<double> GetEstimatedValues(IDataset ds, IEnumerable<int> rows) {
       // lookup columns for variableNames in one pass over the tree to speed up evaluation later on
+      ReadOnlyCollection<double>[] columnCache = new ReadOnlyCollection<double>[tree.Length];
+
       for (int i = 0; i < tree.Length; i++) {
         if (tree[i].VarName != TreeNode.NO_VARIABLE) {
-          tree[i].Data = ds.GetReadOnlyDoubleValues(tree[i].VarName);
-        }
-      }
-      return rows.Select(r => GetPredictionForRow(tree, 0, r));
-    }
-
-    public IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+          // tree models also support calculating estimations if not all variables used for training are available in the dataset
+          if (ds.ColumnNames.Contains(tree[i].VarName))
+            columnCache[i] = ds.GetReadOnlyDoubleValues(tree[i].VarName);
+        }
+      }
+      return rows.Select(r => GetPredictionForRow(tree, columnCache, 0, r));
+    }
+
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
       return new RegressionSolution(this, new RegressionProblemData(problemData));
     }
@@ -147,8 +216,9 @@
       } else {
         return
-          TreeToString(n.LeftIdx, string.Format(CultureInfo.InvariantCulture, "{0}{1}{2} <= {3:F}", part, string.IsNullOrEmpty(part) ? "" : " and ", n.VarName, n.Val))
-        + TreeToString(n.RightIdx, string.Format(CultureInfo.InvariantCulture, "{0}{1}{2} >  {3:F}", part, string.IsNullOrEmpty(part) ? "" : " and ", n.VarName, n.Val));
-      }
-    }
+          TreeToString(n.LeftIdx, string.Format(CultureInfo.InvariantCulture, "{0}{1}{2} <= {3:F} ({4:N3})", part, string.IsNullOrEmpty(part) ? "" : " and ", n.VarName, n.Val, n.WeightLeft))
+        + TreeToString(n.RightIdx, string.Format(CultureInfo.InvariantCulture, "{0}{1}{2}  >  {3:F} ({4:N3}))", part, string.IsNullOrEmpty(part) ? "" : " and ", n.VarName, n.Val, 1.0 - n.WeightLeft));
+      }
+    }
+
   }
 }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj	(revision 14029)
@@ -122,4 +122,8 @@
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="BaselineClassifiers\OneR.cs" />
+    <Compile Include="BaselineClassifiers\OneRClassificationModel.cs" />
+    <Compile Include="BaselineClassifiers\OneRClassificationSolution.cs" />
+    <Compile Include="BaselineClassifiers\ZeroR.cs" />
     <Compile Include="CrossValidation.cs">
       <SubType>Code</SubType>
@@ -174,4 +178,5 @@
     <Compile Include="GaussianProcess\GaussianProcessClassificationModelCreator.cs" />
     <Compile Include="GaussianProcess\GaussianProcessClassification.cs" />
+    <Compile Include="GaussianProcess\MeanFunctions\MeanModel.cs" />
     <Compile Include="GaussianProcess\MeanFunctions\MeanConst.cs" />
     <Compile Include="GaussianProcess\MeanFunctions\MeanLinear.cs" />
@@ -189,4 +194,5 @@
       <SubType>Code</SubType>
     </Compile>
+    <Compile Include="GaussianProcess\GaussianProcessCovarianceOptimizationProblem.cs" />
     <Compile Include="GaussianProcess\Util.cs" />
     <Compile Include="GaussianProcess\IMeanFunction.cs" />
@@ -195,4 +201,6 @@
     <Compile Include="GaussianProcess\GaussianProcessRegressionSolution.cs" />
     <Compile Include="GaussianProcess\ICovarianceFunction.cs" />
+    <Compile Include="GBM\GradientBoostingRegressionAlgorithm.cs" />
+    <Compile Include="GradientBoostedTrees\IGradientBoostedTreesModel.cs" />
     <Compile Include="GradientBoostedTrees\GradientBoostedTreesModelSurrogate.cs" />
     <Compile Include="GradientBoostedTrees\GradientBoostedTreesAlgorithm.cs" />
@@ -200,8 +208,10 @@
     <Compile Include="GradientBoostedTrees\GradientBoostedTreesModel.cs" />
     <Compile Include="GradientBoostedTrees\LossFunctions\AbsoluteErrorLoss.cs" />
+    <Compile Include="GradientBoostedTrees\LossFunctions\QuantileRegressionLoss.cs" />
     <Compile Include="GradientBoostedTrees\LossFunctions\ILossFunction.cs" />
     <Compile Include="GradientBoostedTrees\LossFunctions\LogisticRegressionLoss.cs" />
     <Compile Include="GradientBoostedTrees\LossFunctions\RelativeErrorLoss.cs" />
     <Compile Include="GradientBoostedTrees\LossFunctions\SquaredErrorLoss.cs" />
+    <Compile Include="GradientBoostedTrees\GradientBoostedTreesSolution.cs" />
     <Compile Include="GradientBoostedTrees\RegressionTreeBuilder.cs" />
     <Compile Include="GradientBoostedTrees\RegressionTreeModel.cs" />
@@ -243,4 +253,20 @@
     <Compile Include="Linear\MultinomialLogitClassificationSolution.cs" />
     <Compile Include="Linear\MultinomialLogitModel.cs" />
+    <Compile Include="MctsSymbolicRegression\Automaton.cs" />
+    <Compile Include="MctsSymbolicRegression\CodeGenerator.cs" />
+    <Compile Include="MctsSymbolicRegression\ConstraintHandler.cs" />
+    <Compile Include="MctsSymbolicRegression\Disassembler.cs" />
+    <Compile Include="MctsSymbolicRegression\ExpressionEvaluator.cs" />
+    <Compile Include="MctsSymbolicRegression\MctsSymbolicRegressionAlgorithm.cs" />
+    <Compile Include="MctsSymbolicRegression\MctsSymbolicRegressionStatic.cs" />
+    <Compile Include="MctsSymbolicRegression\OpCodes.cs" />
+    <Compile Include="MctsSymbolicRegression\Policies\EpsGreedy.cs" />
+    <Compile Include="MctsSymbolicRegression\Policies\UcbTuned.cs" />
+    <Compile Include="MctsSymbolicRegression\Policies\IActionStatistics.cs" />
+    <Compile Include="MctsSymbolicRegression\Policies\IPolicy.cs" />
+    <Compile Include="MctsSymbolicRegression\Policies\PolicyBase.cs" />
+    <Compile Include="MctsSymbolicRegression\Policies\Ucb.cs" />
+    <Compile Include="MctsSymbolicRegression\SymbolicExpressionGenerator.cs" />
+    <Compile Include="MctsSymbolicRegression\Tree.cs" />
     <Compile Include="Nca\Initialization\INcaInitializer.cs" />
     <Compile Include="Nca\Initialization\LdaInitializer.cs" />
@@ -272,4 +298,5 @@
     <Compile Include="NeuralNetwork\NeuralNetworkRegression.cs" />
     <Compile Include="NeuralNetwork\NeuralNetworkRegressionSolution.cs" />
+    <Compile Include="NonlinearRegression\NonlinearRegression.cs" />
     <Compile Include="Plugin.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
@@ -300,4 +327,8 @@
       <Private>False</Private>
     </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm\3.3\HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm-3.3.csproj">
+      <Project>{F409DD9E-1E9C-4EB1-AA3A-9F6E987C6E58}</Project>
+      <Name>HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm-3.3</Name>
+    </ProjectReference>
     <ProjectReference Include="..\..\HeuristicLab.Analysis\3.3\HeuristicLab.Analysis-3.3.csproj">
       <Project>{887425B4-4348-49ED-A457-B7D2C26DDBF9}</Project>
@@ -398,4 +429,8 @@
       <Name>HeuristicLab.Random-3.3</Name>
       <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Selection\3.3\HeuristicLab.Selection-3.3.csproj">
+      <Project>{2C36CD4F-E5F5-43A4-801A-201EA895FE17}</Project>
+      <Name>HeuristicLab.Selection-3.3</Name>
     </ProjectReference>
   </ItemGroup>
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearDiscriminantAnalysis.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearDiscriminantAnalysis.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearDiscriminantAnalysis.cs	(revision 14029)
@@ -111,5 +111,5 @@
       IClassificationProblemData problemData,
       IEnumerable<int> rows) {
-      var model = new SymbolicDiscriminantFunctionClassificationModel(tree, interpreter, new AccuracyMaximizationThresholdCalculator());
+      var model = new SymbolicDiscriminantFunctionClassificationModel(problemData.TargetVariable, tree, interpreter, new AccuracyMaximizationThresholdCalculator());
       model.RecalculateModelParameters(problemData, rows);
       return model;
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegression.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegression.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegression.cs	(revision 14029)
@@ -37,5 +37,5 @@
   /// Linear regression data analysis algorithm.
   /// </summary>
-  [Item("Linear Regression", "Linear regression data analysis algorithm (wrapper for ALGLIB).")]
+  [Item("Linear Regression (LR)", "Linear regression data analysis algorithm (wrapper for ALGLIB).")]
   [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 100)]
   [StorableClass]
@@ -110,5 +110,5 @@
       addition.AddSubtree(cNode);
 
-      SymbolicRegressionSolution solution = new SymbolicRegressionSolution(new SymbolicRegressionModel(tree, new SymbolicDataAnalysisExpressionTreeInterpreter()), (IRegressionProblemData)problemData.Clone());
+      SymbolicRegressionSolution solution = new SymbolicRegressionSolution(new SymbolicRegressionModel(problemData.TargetVariable, tree, new SymbolicDataAnalysisExpressionTreeInterpreter()), (IRegressionProblemData)problemData.Clone());
       solution.Model.Name = "Linear Regression Model";
       solution.Name = "Linear Regression Solution";
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassification.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassification.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassification.cs	(revision 14029)
@@ -34,5 +34,5 @@
   /// Multinomial logit regression data analysis algorithm.
   /// </summary>
-  [Item("Multinomial Logit Classification", "Multinomial logit classification data analysis algorithm (wrapper for ALGLIB).")]
+  [Item("Multinomial Logit Classification (MNL)", "Multinomial logit classification data analysis algorithm (wrapper for ALGLIB).")]
   [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 180)]
   [StorableClass]
@@ -95,5 +95,5 @@
       relClassError = alglib.mnlrelclserror(lm, inputMatrix, nRows);
 
-      MultinomialLogitClassificationSolution solution = new MultinomialLogitClassificationSolution((IClassificationProblemData)problemData.Clone(), new MultinomialLogitModel(lm, targetVariable, allowedInputVariables, classValues));
+      MultinomialLogitClassificationSolution solution = new MultinomialLogitClassificationSolution(new MultinomialLogitModel(lm, targetVariable, allowedInputVariables, classValues), (IClassificationProblemData)problemData.Clone());
       return solution;
     }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassificationSolution.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassificationSolution.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassificationSolution.cs	(revision 14029)
@@ -43,5 +43,5 @@
       : base(original, cloner) {
     }
-    public MultinomialLogitClassificationSolution(IClassificationProblemData problemData, MultinomialLogitModel logitModel)
+    public MultinomialLogitClassificationSolution( MultinomialLogitModel logitModel,IClassificationProblemData problemData)
       : base(logitModel, problemData) {
     }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitModel.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitModel.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitModel.cs	(revision 14029)
@@ -34,5 +34,5 @@
   [StorableClass]
   [Item("Multinomial Logit Model", "Represents a multinomial logit model for classification.")]
-  public sealed class MultinomialLogitModel : NamedItem, IClassificationModel {
+  public sealed class MultinomialLogitModel : ClassificationModel {
 
     private alglib.logitmodel logitModel;
@@ -48,6 +48,8 @@
     }
 
-    [Storable]
-    private string targetVariable;
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return allowedInputVariables; }
+    }
+
     [Storable]
     private string[] allowedInputVariables;
@@ -64,14 +66,12 @@
       logitModel = new alglib.logitmodel();
       logitModel.innerobj.w = (double[])original.logitModel.innerobj.w.Clone();
-      targetVariable = original.targetVariable;
       allowedInputVariables = (string[])original.allowedInputVariables.Clone();
       classValues = (double[])original.classValues.Clone();
     }
     public MultinomialLogitModel(alglib.logitmodel logitModel, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues)
-      : base() {
+      : base(targetVariable) {
       this.name = ItemName;
       this.description = ItemDescription;
       this.logitModel = logitModel;
-      this.targetVariable = targetVariable;
       this.allowedInputVariables = allowedInputVariables.ToArray();
       this.classValues = (double[])classValues.Clone();
@@ -82,5 +82,5 @@
     }
 
-    public IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
       double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
 
@@ -108,9 +108,6 @@
     }
 
-    public MultinomialLogitClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
-      return new MultinomialLogitClassificationSolution(new ClassificationProblemData(problemData), this);
-    }
-    IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {
-      return CreateClassificationSolution(problemData);
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new MultinomialLogitClassificationSolution(this, new ClassificationProblemData(problemData));
     }
 
@@ -135,4 +132,5 @@
     }
     #endregion
+
   }
 }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Automaton.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Automaton.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Automaton.cs	(revision 14029)
@@ -0,0 +1,409 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression {
+  // this is the core class for generating expressions. 
+  // it represents a finite state automaton, each state transition can be associated with an action (e.g. to produce code).
+  // the automaton determines the possible structures for expressions.
+  // 
+  // to understand this code it is worthwile to generate a graphical visualization of the automaton (see PrintAutomaton).
+  // If the code is compiled in debug mode the automaton produces a Graphviz file into the folder of the application
+  // whenever an instance of the automaton is constructed.
+  //
+  // This class relies on two other classes: 
+  // - CodeGenerator to produce code for a stack-based evaluator and
+  // - ConstraintHandler to restrict the allowed set of expressions. 
+  //
+  // The ConstraintHandler extends the automaton and adds semantic restrictions for expressions produced by the automaton.
+  // 
+  // 
+  internal class Automaton {
+    public const int StateExpr = 1;
+    public const int StateExprEnd = 2;
+    public const int StateTermStart = 3;
+    public const int StateTermEnd = 4;
+    public const int StateFactorStart = 5;
+    public const int StateFactorEnd = 6;
+    public const int StateVariableFactorStart = 7;
+    public const int StateVariableFactorEnd = 8;
+    public const int StateExpFactorStart = 9;
+    public const int StateExpFactorEnd = 10;
+    public const int StateLogFactorStart = 11;
+    public const int StateLogFactorEnd = 12;
+    public const int StateInvFactorStart = 13;
+    public const int StateInvFactorEnd = 14;
+    public const int StateExpFStart = 15;
+    public const int StateExpFEnd = 16;
+    public const int StateLogTStart = 17;
+    public const int StateLogTEnd = 18;
+    public const int StateLogTFStart = 19;
+    public const int StateLogTFEnd = 20;
+    public const int StateInvTStart = 21;
+    public const int StateInvTEnd = 22;
+    public const int StateInvTFStart = 23;
+    public const int StateInvTFEnd = 24;
+    public const int FirstDynamicState = 25;
+    // more states for individual variables are created dynamically
+
+    private const int StartState = StateExpr;
+    public int CurrentState { get; private set; }
+
+    public readonly List<string> stateNames;
+    private List<int>[] followStates;
+    private List<Action>[,] actions; // not every follow state is possible but this representation should be efficient
+    private List<string>[,] actionStrings; // just for printing 
+    private readonly CodeGenerator codeGenerator;
+    private readonly ConstraintHandler constraintHandler;
+
+    public Automaton(double[][] vars, int maxVarsInExpression = 100,
+       bool allowProdOfVars = true,
+       bool allowExp = true,
+       bool allowLog = true,
+       bool allowInv = true,
+       bool allowMultipleTerms = false) {
+      int nVars = vars.Length;
+      stateNames = new List<string>() { string.Empty, "Expr", "ExprEnd", "TermStart", "TermEnd", "FactorStart", "FactorEnd", "VarFactorStart", "VarFactorEnd", "ExpFactorStart", "ExpFactorEnd", "LogFactorStart", "LogFactorEnd", "InvFactorStart", "InvFactorEnd", "ExpFStart", "ExpFEnd", "LogTStart", "LogTEnd", "LogTFStart", "LogTFEnd", "InvTStart", "InvTEnd", "InvTFStart", "InvTFEnd" };
+      codeGenerator = new CodeGenerator();
+      constraintHandler = new ConstraintHandler(maxVarsInExpression);
+      BuildAutomaton(nVars, allowProdOfVars, allowExp, allowLog, allowInv, allowMultipleTerms);
+
+      Reset();
+#if DEBUG
+      PrintAutomaton();
+#endif
+    }
+
+    // reverse notation ops
+    // Expr -> c 0 Term { '+' Term } '+' '*' c '+' 'exit'
+    // Term -> c Fact { '*' Fact } '*'
+    // Fact -> VarFact | ExpFact | LogFact | InvFact
+    // VarFact -> var_1 ... var_n
+    // ExpFact -> 1 ExpF { '*' ExpF } '*' c '*' 'exp' // c must be at end to allow scaling in evaluator
+    // ExpF    -> var_1 ... var_n 
+    // LogFact -> 0 LogT { '+' LogT } '+' c '+' 'log' // c must be at end to allow scaling in evaluator
+    // LogT    -> c LogTF { '*' LogTF } '*' 
+    // LogTF   -> var_1 ... var_n 
+    // InvFact -> 1 InvT { '+' InvT } '+' 'inv'
+    // InvT    -> (var_1 ... var_n) c '*' 
+    private void BuildAutomaton(int nVars,
+      bool allowProdOfVars = true,
+       bool allowExp = true,
+       bool allowLog = true,
+       bool allowInv = true,
+       bool allowMultipleTerms = false) {
+
+      int nStates = FirstDynamicState + 4 * nVars;
+      followStates = new List<int>[nStates];
+      actions = new List<Action>[nStates, nStates];
+      actionStrings = new List<string>[nStates, nStates];
+
+      // Expr -> c 0 Term { '+' Term } '+' '*' c '+' 'exit'
+      AddTransition(StateExpr, StateTermStart, () => {
+        codeGenerator.Reset();
+        codeGenerator.Emit1(OpCodes.LoadParamN);
+        codeGenerator.Emit1(OpCodes.LoadConst0);
+        constraintHandler.Reset();
+      }, "c 0, Reset");
+      AddTransition(StateTermEnd, StateExprEnd, () => {
+        codeGenerator.Emit1(OpCodes.Add);
+        codeGenerator.Emit1(OpCodes.Mul);
+        codeGenerator.Emit1(OpCodes.LoadParamN);
+        codeGenerator.Emit1(OpCodes.Add);
+        codeGenerator.Emit1(OpCodes.Exit);
+      }, "+*c+ exit");
+      if (allowMultipleTerms)
+        AddTransition(StateTermEnd, StateTermStart, () => {
+          codeGenerator.Emit1(OpCodes.Add);
+        }, "+");
+
+      // Term -> c Fact { '*' Fact } '*'
+      AddTransition(StateTermStart, StateFactorStart,
+        () => {
+          codeGenerator.Emit1(OpCodes.LoadParamN);
+          constraintHandler.StartTerm();
+        },
+        "c, StartTerm");
+      AddTransition(StateFactorEnd, StateTermEnd,
+        () => {
+          codeGenerator.Emit1(OpCodes.Mul);
+          constraintHandler.EndTerm();
+        },
+        "*, EndTerm");
+
+      AddTransition(StateFactorEnd, StateFactorStart,
+        () => { codeGenerator.Emit1(OpCodes.Mul); },
+        "*");
+
+
+      // Fact -> VarFact | ExpFact | LogFact | InvFact
+      if (allowProdOfVars)
+        AddTransition(StateFactorStart, StateVariableFactorStart, () => {
+          constraintHandler.StartFactor(StateVariableFactorStart);
+        }, "StartFactor");
+      if (allowExp)
+        AddTransition(StateFactorStart, StateExpFactorStart, () => {
+          constraintHandler.StartFactor(StateExpFactorStart);
+        }, "StartFactor");
+      if (allowLog)
+        AddTransition(StateFactorStart, StateLogFactorStart, () => {
+          constraintHandler.StartFactor(StateLogFactorStart);
+        }, "StartFactor");
+      if (allowInv)
+        AddTransition(StateFactorStart, StateInvFactorStart, () => {
+          constraintHandler.StartFactor(StateInvFactorStart);
+        }, "StartFactor");
+      AddTransition(StateVariableFactorEnd, StateFactorEnd, () => { constraintHandler.EndFactor(); }, "EndFactor");
+      AddTransition(StateExpFactorEnd, StateFactorEnd, () => { constraintHandler.EndFactor(); }, "EndFactor");
+      AddTransition(StateLogFactorEnd, StateFactorEnd, () => { constraintHandler.EndFactor(); }, "EndFactor");
+      AddTransition(StateInvFactorEnd, StateFactorEnd, () => { constraintHandler.EndFactor(); }, "EndFactor");
+
+      // VarFact -> var_1 ... var_n
+      // add dynamic states for each variable 
+      int curDynVarState = FirstDynamicState;
+      for (int i = 0; i < nVars; i++) {
+        short varIdx = (short)i;
+        var varState = curDynVarState;
+        stateNames.Add("var_1");
+        AddTransition(StateVariableFactorStart, curDynVarState,
+          () => {
+            codeGenerator.Emit2(OpCodes.LoadVar, varIdx);
+            constraintHandler.AddVarToCurrentFactor(varState);
+          },
+          "var_" + varIdx + ", AddVar");
+        AddTransition(curDynVarState, StateVariableFactorEnd);
+        curDynVarState++;
+      }
+
+      // ExpFact -> 1 ExpF { '*' ExpF } '*' c '*' 'exp' 
+      AddTransition(StateExpFactorStart, StateExpFStart,
+        () => {
+          codeGenerator.Emit1(OpCodes.LoadConst1);
+        },
+        "1");
+      AddTransition(StateExpFEnd, StateExpFactorEnd,
+        () => {
+          codeGenerator.Emit1(OpCodes.Mul);
+          codeGenerator.Emit1(OpCodes.LoadParamN);
+          codeGenerator.Emit1(OpCodes.Mul);
+          codeGenerator.Emit1(OpCodes.Exp);
+        },
+        "*c*exp");
+      AddTransition(StateExpFEnd, StateExpFStart,
+        () => { codeGenerator.Emit1(OpCodes.Mul); },
+        "*");
+
+      // ExpF    -> var_1 ... var_n 
+      for (int i = 0; i < nVars; i++) {
+        short varIdx = (short)i;
+        int varState = curDynVarState;
+        stateNames.Add("var_2");
+        AddTransition(StateExpFStart, curDynVarState,
+          () => {
+            codeGenerator.Emit2(OpCodes.LoadVar, varIdx);
+            constraintHandler.AddVarToCurrentFactor(varState);
+          },
+          "var_" + varIdx + ", AddVar");
+        AddTransition(curDynVarState, StateExpFEnd);
+        curDynVarState++;
+      }
+
+      // must have c at end because of adjustment of c in evaluator
+      // LogFact -> 0 LogT { '+' LogT } '+' c '+' 'log'
+      AddTransition(StateLogFactorStart, StateLogTStart,
+        () => {
+          codeGenerator.Emit1(OpCodes.LoadConst0);
+          constraintHandler.StartNewTermInPoly();
+        },
+        "0, StartTermInPoly");
+      AddTransition(StateLogTEnd, StateLogFactorEnd,
+        () => {
+          codeGenerator.Emit1(OpCodes.Add);
+          codeGenerator.Emit1(OpCodes.LoadParamN);
+          codeGenerator.Emit1(OpCodes.Add);
+          codeGenerator.Emit1(OpCodes.Log);
+        },
+        "+c+log");
+      AddTransition(StateLogTEnd, StateLogTStart,
+        () => { codeGenerator.Emit1(OpCodes.Add); },
+        "+");
+
+      // LogT    -> c LogTF { '*' LogTF } '*' 
+      AddTransition(StateLogTStart, StateLogTFStart,
+        () => {
+          codeGenerator.Emit1(OpCodes.LoadParamN);
+        },
+        "c");
+      AddTransition(StateLogTFEnd, StateLogTEnd,
+        () => {
+          codeGenerator.Emit1(OpCodes.Mul);
+        },
+        "*");
+      AddTransition(StateLogTFEnd, StateLogTFStart,
+        () => {
+          codeGenerator.Emit1(OpCodes.Mul);
+        },
+        "*");
+
+      // LogTF   -> var_1 ... var_n
+      for (int i = 0; i < nVars; i++) {
+        short varIdx = (short)i;
+        int varState = curDynVarState;
+        stateNames.Add("var_3");
+        AddTransition(StateLogTFStart, curDynVarState,
+          () => {
+            codeGenerator.Emit2(OpCodes.LoadVar, varIdx);
+            constraintHandler.AddVarToCurrentFactor(varState);
+          },
+          "var_" + varIdx + ", AddVar");
+        AddTransition(curDynVarState, StateLogTFEnd);
+        curDynVarState++;
+      }
+
+      // InvFact -> 1 InvT { '+' InvT } '+' 'inv'
+      AddTransition(StateInvFactorStart, StateInvTStart,
+        () => {
+          codeGenerator.Emit1(OpCodes.LoadConst1);
+          constraintHandler.StartNewTermInPoly();
+        },
+        "c, StartTermInPoly");
+      AddTransition(StateInvTEnd, StateInvFactorEnd,
+        () => {
+          codeGenerator.Emit1(OpCodes.Add);
+          codeGenerator.Emit1(OpCodes.Inv);
+        },
+        "+inv");
+      AddTransition(StateInvTEnd, StateInvTStart,
+        () => { codeGenerator.Emit1(OpCodes.Add); },
+        "+");
+
+      // InvT    -> c InvTF { '*' InvTF } '*' 
+      AddTransition(StateInvTStart, StateInvTFStart,
+        () => {
+          codeGenerator.Emit1(OpCodes.LoadParamN);
+        },
+        "c");
+      AddTransition(StateInvTFEnd, StateInvTEnd,
+        () => {
+          codeGenerator.Emit1(OpCodes.Mul);
+        },
+        "*");
+      AddTransition(StateInvTFEnd, StateInvTFStart,
+        () => {
+          codeGenerator.Emit1(OpCodes.Mul);
+        },
+        "*");
+
+      // InvTF    -> (var_1 ... var_n) c '*' 
+      for (int i = 0; i < nVars; i++) {
+        short varIdx = (short)i;
+        int varState = curDynVarState;
+        stateNames.Add("var_4");
+        AddTransition(StateInvTFStart, curDynVarState,
+          () => {
+            codeGenerator.Emit2(OpCodes.LoadVar, varIdx);
+            constraintHandler.AddVarToCurrentFactor(varState);
+          },
+          "var_" + varIdx + ", AddVar");
+        AddTransition(curDynVarState, StateInvTFEnd);
+        curDynVarState++;
+      }
+
+      followStates[StateExprEnd] = new List<int>(); // no follow states
+    }
+
+    private void AddTransition(int fromState, int toState) {
+      if (followStates[fromState] == null) followStates[fromState] = new List<int>();
+      followStates[fromState].Add(toState);
+    }
+    private void AddTransition(int fromState, int toState, Action action, string str) {
+      if (followStates[fromState] == null) followStates[fromState] = new List<int>();
+      followStates[fromState].Add(toState);
+
+      if (actions[fromState, toState] == null) {
+        actions[fromState, toState] = new List<Action>();
+        actionStrings[fromState, toState] = new List<string>();
+      }
+
+      actions[fromState, toState].Add(action);
+      actionStrings[fromState, toState].Add(str);
+    }
+
+    private readonly int[] followStatesBuf = new int[1000];
+    public void FollowStates(int state, out int[] buf, out int nElements) {
+      // for loop instead of where iterator
+      var fs = followStates[state];
+      int j = 0;
+      for (int i = 0; i < fs.Count; i++) {
+        var s = fs[i];
+        if (constraintHandler.IsAllowedFollowState(state, s)) {
+          followStatesBuf[j++] = s;
+        }
+      }
+      buf = followStatesBuf;
+      nElements = j;
+    }
+
+
+    public void Goto(int targetState) {
+      if (actions[CurrentState, targetState] != null)
+        actions[CurrentState, targetState].ForEach(a => a()); // execute all actions
+      CurrentState = targetState;
+    }
+
+    public bool IsFinalState(int s) {
+      return s == StateExprEnd && !constraintHandler.IsInvalidExpression;
+    }
+
+    public void GetCode(out byte[] code, out int nParams) {
+      codeGenerator.GetCode(out code, out nParams);
+    }
+
+    public void Reset() {
+      CurrentState = StartState;
+      codeGenerator.Reset();
+      constraintHandler.Reset();
+    }
+
+#if DEBUG
+    public void PrintAutomaton() {
+      using (var writer = new StreamWriter("automaton.gv")) {
+        writer.WriteLine("digraph {");
+        // writer.WriteLine("rankdir=LR");
+        int[] fs;
+        int nFs;
+        for (int s = StartState; s < stateNames.Count; s++) {
+          for (int i = 0; i < followStates[s].Count; i++) {
+            if (followStates[s][i] <= 0) continue;
+            var followS = followStates[s][i];
+            var label = actionStrings[s, followS] != null ? string.Join(" , ", actionStrings[s, followS]) : "";
+            writer.WriteLine("{0} -> {1} [ label = \"{2}\" ];", stateNames[s], stateNames[followS], label);
+          }
+        }
+        writer.WriteLine("}");
+      }
+    }
+#endif
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/CodeGenerator.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/CodeGenerator.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/CodeGenerator.cs	(revision 14029)
@@ -0,0 +1,56 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Diagnostics.Contracts;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression {
+  internal class CodeGenerator {
+
+    public const int MaxCodeLength = 10000;
+    public const int MaxConstants = 100;
+
+    private readonly byte[] codeArray = new byte[MaxCodeLength];
+    private int pc;
+    private int nParams;
+
+    public void Reset() {
+      pc = 0;
+      nParams = 0;
+    }
+
+    public void Emit1(OpCodes code) {
+      codeArray[pc++] = (byte)code;
+      if (code == OpCodes.LoadParamN) nParams++;
+    }
+
+    public void Emit2(OpCodes code, short arg) {
+      Contract.Assert(code == OpCodes.LoadVar); // only loadVar opcode has params
+      Emit1(code);
+      codeArray[pc++] = (byte)(arg >> 8);
+      codeArray[pc++] = (byte)(arg & 0xFF);
+    }
+
+    public void GetCode(out byte[] code, out int nParams) {
+      code = codeArray;
+      nParams = this.nParams;
+    }
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/ConstraintHandler.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/ConstraintHandler.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/ConstraintHandler.cs	(revision 14029)
@@ -0,0 +1,417 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.Contracts;
+using System.Linq;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression {
+
+  // This class restricts the set of allowed transitions of the automaton to prevent exploration of duplicate expressions.
+  // It would be possible to implement this class in such a way that the search never visits a duplicate expression. However,
+  // it seems very intricate to detect this robustly and in all cases while generating an expression because 
+  // some for of lookahead is necessary. 
+  // Instead the constraint handler only catches the obvious duplicates directly, but does not guarantee that the search always produces a valid expression.
+  // The ratio of the number of unsuccessful searches, that need backtracking should be tracked in the MCTS alg (MctsSymbolicRegressionStatic)
+
+  // All changes to this class should be tested through unit tests. It is important that the ConstraintHandler is not too restrictive.
+
+  // the constraints are derived from a canonical form for expressions.
+  // overall we can enforce a limited number of variable references
+  // 
+  // an expression is a sum of terms t_1 ... t_n where terms are ordered according to a relation t_i (<=)_term t_j for each pair t_i, t_j and i <= j
+  // a term is a product of factors where factors are ordered according to relation f_i (<=)_factor f_j for each pair f_i,f_j and i <= j
+
+  // we want to enforce lower-order terms before higher-order terms in expressions (based on number of variable references)
+  // factors can have different types (variable, exp, log, inverse)
+
+  // (<=)_term  [IsSmallerOrEqualTerm(t_i, t_j)]
+  //   1.  NumberOfVarRefs(t_i) < NumberOfVarRefs(t_j)  --> true           enforce terms with non-decreasing number of var refs
+  //   2.  NumberOfVarRefs(t_i) > NumberOfVarRefs(t_j)  --> false
+  //   3.  NumFactors(t_i) > NumFactors(t_j)            --> true           enforce terms with non-increasing number of factors
+  //   4.  NumFactors(t_i) < NumFactors(t_j)            --> false
+  //   5.  for all k factors: Factor(k, t_i) (<=)_factor  Factor(k, t_j) --> true // factors must be non-decreasing
+  //   6.  all factors are (=)_factor                   --> true
+  //   7.  else false
+
+  // (<=)_factor  [IsSmallerOrEqualFactor(f_i, f_j)]
+  //   1.  FactorType(t_i) < FactorType(t_j)  --> true           enforce terms with non-decreasing factor type (var < exp < log < inv)
+  //   2.  FactorType(t_i) > FactorType(t_j)  --> false
+  //   3.  Compare the two factors specifically
+  //     - variables: varIdx_i <= varIdx_j (only one var reference)
+  //     - exp: number of variable references and then varIdx_i <= varIdx_j for each position
+  //     - log: number of variable references and ...
+  //     - inv: number of variable references and ...
+  //
+
+  // for log and inverse factors we allow all polynomials as argument
+  // a polynomial is a sum of terms t_1 ... t_n where terms are ordered according to a relation t_i (<=)_poly t_j for each pair t_i, t_j and i <= j
+
+  // (<=)_poly  [IsSmallerOrEqualPoly(t_i, t_j)]
+  //  1. NumberOfVarRefs(t_i) < NumberOfVarRefs(t_j)         --> true // enforce non-decreasing number of var refs
+  //  2. NumberOfVarRefs(t_i) > NumberOfVarRefs(t_j)         --> false // enforce non-decreasing number of var refs
+  //  3. for all k variables: VarIdx(k,t_i) > VarIdx(k, t_j) --> false // enforce non-decreasing variable idx
+
+
+  // we store the following to make comparsions:
+  // - prevTerm (complete & containing all factors)
+  // - curTerm  (incomplete & containing all completed factors)
+  // - curFactor (incomplete)
+  internal class ConstraintHandler {
+    private int nVars;
+    private readonly int maxVariables;
+    private bool invalidExpression;
+
+    public bool IsInvalidExpression {
+      get { return invalidExpression; }
+    }
+
+
+    private TermInformation prevTerm;
+    private TermInformation curTerm;
+    private FactorInformation curFactor;
+
+
+    private class TermInformation {
+      public int numVarReferences { get { return factors.Sum(f => f.numVarReferences); } }
+      public List<FactorInformation> factors = new List<FactorInformation>();
+    }
+
+    private class FactorInformation {
+      public int numVarReferences = 0;
+      public int factorType; // use the state number to represent types
+
+      // for variable factors
+      public int variableState = -1;
+
+      // for exp  factors
+      public List<int> expVariableStates = new List<int>();
+
+      // for log and inv factors
+      public List<List<int>> polyVariableStates = new List<List<int>>();
+    }
+
+
+    public ConstraintHandler(int maxVars) {
+      this.maxVariables = maxVars;
+    }
+
+    // the order relations for terms and factors
+
+    private static int CompareTerms(TermInformation a, TermInformation b) {
+      if (a.numVarReferences < b.numVarReferences) return -1;
+      if (a.numVarReferences > b.numVarReferences) return 1;
+
+      if (a.factors.Count > b.factors.Count) return -1;  // terms with more factors should be ordered first
+      if (a.factors.Count < b.factors.Count) return +1;
+
+      var aFactors = a.factors.GetEnumerator();
+      var bFactors = b.factors.GetEnumerator();
+      while (aFactors.MoveNext() & bFactors.MoveNext()) {
+        var c = CompareFactors(aFactors.Current, bFactors.Current);
+        if (c < 0) return -1;
+        if (c > 0) return 1;
+      }
+      // all factors are the same => terms are the same
+      return 0;
+    }
+
+    private static int CompareFactors(FactorInformation a, FactorInformation b) {
+      if (a.factorType < b.factorType) return -1;
+      if (a.factorType > b.factorType) return +1;
+      // same factor types
+      if (a.factorType == Automaton.StateVariableFactorStart) {
+        return a.variableState.CompareTo(b.variableState);
+      } else if (a.factorType == Automaton.StateExpFactorStart) {
+        return CompareStateLists(a.expVariableStates, b.expVariableStates);
+      } else {
+        if (a.numVarReferences < b.numVarReferences) return -1;
+        if (a.numVarReferences > b.numVarReferences) return +1;
+        if (a.polyVariableStates.Count > b.polyVariableStates.Count) return -1; // more terms in the poly should be ordered first
+        if (a.polyVariableStates.Count < b.polyVariableStates.Count) return +1;
+        // log and inv
+        var aTerms = a.polyVariableStates.GetEnumerator();
+        var bTerms = b.polyVariableStates.GetEnumerator();
+        while (aTerms.MoveNext() & bTerms.MoveNext()) {
+          var c = CompareStateLists(aTerms.Current, bTerms.Current);
+          if (c != 0) return c;
+        }
+        return 0; // all terms in the polynomial are the same
+      }
+    }
+
+    private static int CompareStateLists(List<int> a, List<int> b) {
+      if (a.Count < b.Count) return -1;
+      if (a.Count > b.Count) return +1;
+      for (int i = 0; i < a.Count; i++) {
+        if (a[i] < b[i]) return -1;
+        if (a[i] > b[i]) return +1;
+      }
+      return 0; // all states are the same
+    }
+
+
+    private bool IsNewTermAllowed() {
+      // next term must have at least as many variable references as the previous term
+      return prevTerm == null || nVars + prevTerm.numVarReferences <= maxVariables;
+    }
+
+    private bool IsNewFactorAllowed() {
+      // next factor must have a larger or equal type compared to the previous factor.
+      // if the types are the same it must have at least as many variable references.
+      // so if the prevFactor is any other than invFactor (last possible type) then we only need to be able to add one variable
+      // otherwise we need to be able to add at least as many variables as the previous factor
+      return !curTerm.factors.Any() ||
+             (nVars + curTerm.factors.Last().numVarReferences <= maxVariables);
+    }
+
+    private bool IsAllowedAsNextFactorType(int followState) {
+      // IsNewTermAllowed already ensures that we can add a term with enough variable references
+
+      // enforce constraints within terms (compare to prev factor)
+      if (curTerm.factors.Any()) {
+        // enforce non-decreasing factor types
+        if (curTerm.factors.Last().factorType > followState) return false;
+        // when the factor type is the same, starting a new factor is only allowed if we can add at least the number of variables of the prev factor
+        if (curTerm.factors.Last().factorType == followState && nVars + curTerm.factors.Last().numVarReferences > maxVariables) return false;
+      }
+
+      // enforce constraints on terms (compare to prev term)
+      // meaning that we must ensure non-decreasing terms
+      if (prevTerm != null) {
+        // a factor type is only allowed if we can then produce a term that is larger or equal to the prev term
+        // (1) if we the number of variable references still remaining is larger than the number of variable references in the prev term 
+        //     then it is always possible to build a larger term
+        // (2) otherwise we try to build the largest possible term starting from current factors in the term.
+        //     
+
+        var numVarRefsRemaining = maxVariables - nVars;
+        Contract.Assert(!curTerm.factors.Any() || curTerm.factors.Last().numVarReferences <= numVarRefsRemaining);
+
+        if (prevTerm.numVarReferences < numVarRefsRemaining) return true;
+
+        // variable factors must be handled differently because they can only contain one variable reference
+        if (followState == Automaton.StateVariableFactorStart) {
+          // append the variable factor and the maximum possible state from the previous factor to create a larger factor
+          var varF = CreateLargestPossibleFactor(Automaton.StateVariableFactorStart, 1);
+          var maxF = CreateLargestPossibleFactor(prevTerm.factors.Max(f => f.factorType), numVarRefsRemaining - 1);
+          var origFactorCount = curTerm.factors.Count;
+          // add this factor to the current term
+          curTerm.factors.Add(varF);
+          curTerm.factors.Add(maxF);
+          var c = CompareTerms(prevTerm, curTerm);
+          // restore term
+          curTerm.factors.RemoveRange(origFactorCount, 2);
+          // if the prev term is still larger then this followstate is not allowed
+          if (c > 0) {
+            return false;
+          }
+        } else {
+          var newF = CreateLargestPossibleFactor(followState, numVarRefsRemaining);
+
+          var origFactorCount = curTerm.factors.Count;
+          // add this factor to the current term
+          curTerm.factors.Add(newF);
+          var c = CompareTerms(prevTerm, curTerm);
+          // restore term
+          curTerm.factors.RemoveAt(origFactorCount);
+          // if the prev term is still larger then this followstate is not allowed
+          if (c > 0) {
+            return false;
+          }
+        }
+      }
+      return true;
+    }
+
+    // largest possible factor of the given kind
+    private FactorInformation CreateLargestPossibleFactor(int factorType, int numVarRefs) {
+      var newF = new FactorInformation();
+      newF.factorType = factorType;
+      if (factorType == Automaton.StateVariableFactorStart) {
+        newF.variableState = int.MaxValue;
+        newF.numVarReferences = 1;
+      } else if (factorType == Automaton.StateExpFactorStart) {
+        for (int i = 0; i < numVarRefs; i++)
+          newF.expVariableStates.Add(int.MaxValue);
+        newF.numVarReferences = numVarRefs;
+      } else if (factorType == Automaton.StateInvFactorStart || factorType == Automaton.StateLogFactorStart) {
+        for (int i = 0; i < numVarRefs; i++) {
+          newF.polyVariableStates.Add(new List<int>());
+          newF.polyVariableStates[i].Add(int.MaxValue);
+        }
+        newF.numVarReferences = numVarRefs;
+      }
+      return newF;
+    }
+
+    private bool IsAllowedAsNextVariableFactor(int variableState) {
+      Contract.Assert(variableState >= Automaton.FirstDynamicState);
+      return !curTerm.factors.Any() || curTerm.factors.Last().variableState <= variableState;
+    }
+
+    private bool IsAllowedAsNextInExp(int variableState) {
+      Contract.Assert(variableState >= Automaton.FirstDynamicState);
+      if (curFactor.expVariableStates.Any() && curFactor.expVariableStates.Last() > variableState) return false;
+      if (curTerm.factors.Any()) {
+        // try and compare with prev factor      
+        curFactor.numVarReferences++;
+        curFactor.expVariableStates.Add(variableState);
+        var c = CompareFactors(curTerm.factors.Last(), curFactor);
+        curFactor.numVarReferences--;
+        curFactor.expVariableStates.RemoveAt(curFactor.expVariableStates.Count - 1);
+        return c <= 0;
+      }
+      return true;
+    }
+
+    private bool IsNewTermAllowedInPoly() {
+      return nVars + curFactor.polyVariableStates.Last().Count() <= maxVariables;
+    }
+
+    private bool IsAllowedAsNextInPoly(int variableState) {
+      Contract.Assert(variableState >= Automaton.FirstDynamicState);
+      return !curFactor.polyVariableStates.Any() ||
+             !curFactor.polyVariableStates.Last().Any() ||
+              curFactor.polyVariableStates.Last().Last() <= variableState;
+    }
+    private bool IsTermCompleteInPoly() {
+      var nTerms = curFactor.polyVariableStates.Count;
+      return nTerms == 1 ||
+             curFactor.polyVariableStates[nTerms - 2].Count <= curFactor.polyVariableStates[nTerms - 1].Count;
+
+    }
+    private bool IsCompleteExp() {
+      return !curTerm.factors.Any() || CompareFactors(curTerm.factors.Last(), curFactor) <= 0;
+    }
+
+    public bool IsAllowedFollowState(int currentState, int followState) {
+      // an invalid action was taken earlier on => nothing can be done anymore
+      if (invalidExpression) return false;
+      // states that have no alternative are always allowed
+      // some ending states are only allowed if enough variables have been used in the term
+      if (
+        currentState == Automaton.StateTermStart ||           // no alternative
+        currentState == Automaton.StateExpFactorStart ||
+        currentState == Automaton.StateLogFactorStart ||
+        currentState == Automaton.StateInvFactorStart ||
+        followState == Automaton.StateVariableFactorEnd ||    // no alternative
+        followState == Automaton.StateExpFEnd ||              // no alternative
+        followState == Automaton.StateLogTFEnd ||             // no alternative
+        followState == Automaton.StateInvTFEnd ||             // no alternative
+        followState == Automaton.StateFactorEnd ||            // always allowed because no alternative
+        followState == Automaton.StateExprEnd                 // we could also constrain the minimum number of terms here
+      ) return true;
+
+
+      // starting a new term is only allowed if we can add a term with at least the number of variables of the prev term
+      if (followState == Automaton.StateTermStart && !IsNewTermAllowed()) return false;
+      if (followState == Automaton.StateFactorStart && !IsNewFactorAllowed()) return false;
+      if (currentState == Automaton.StateFactorStart && !IsAllowedAsNextFactorType(followState)) return false;
+      if (followState == Automaton.StateTermEnd && prevTerm != null && CompareTerms(prevTerm, curTerm) > 0) return false;
+
+      // all of these states add at least one variable
+      if (
+          followState == Automaton.StateVariableFactorStart ||
+          followState == Automaton.StateExpFactorStart || followState == Automaton.StateExpFStart ||
+          followState == Automaton.StateLogFactorStart || followState == Automaton.StateLogTStart ||
+          followState == Automaton.StateLogTFStart ||
+          followState == Automaton.StateInvFactorStart || followState == Automaton.StateInvTStart ||
+          followState == Automaton.StateInvTFStart) {
+        if (nVars + 1 > maxVariables) return false;
+      }
+
+      if (currentState == Automaton.StateVariableFactorStart && !IsAllowedAsNextVariableFactor(followState)) return false;
+      else if (currentState == Automaton.StateExpFStart && !IsAllowedAsNextInExp(followState)) return false;
+      else if (followState == Automaton.StateLogTStart && !IsNewTermAllowedInPoly()) return false;
+      else if (currentState == Automaton.StateLogTFStart && !IsAllowedAsNextInPoly(followState)) return false;
+      else if (followState == Automaton.StateInvTStart && !IsNewTermAllowedInPoly()) return false;
+      else if (currentState == Automaton.StateInvTFStart && !IsAllowedAsNextInPoly(followState)) return false;
+      // finishing an exponential factor is only allowed when the number of variable references is large enough
+      else if (followState == Automaton.StateExpFactorEnd && !IsCompleteExp()) return false;
+      // finishing a polynomial (in log or inv) is only allowed when the number of variable references is large enough
+      else if (followState == Automaton.StateInvTEnd && !IsTermCompleteInPoly()) return false;
+      else if (followState == Automaton.StateLogTEnd && !IsTermCompleteInPoly()) return false;
+
+      else if (nVars > maxVariables) return false;
+      else return true;
+    }
+
+
+    public void Reset() {
+      nVars = 0;
+      prevTerm = null;
+      curTerm = null;
+      curFactor = null;
+      invalidExpression = false;
+    }
+
+    public void StartTerm() {
+      curTerm = new TermInformation();
+    }
+
+    public void StartFactor(int state) {
+      curFactor = new FactorInformation();
+      curFactor.factorType = state;
+    }
+
+
+    public void AddVarToCurrentFactor(int state) {
+      Contract.Assert(Automaton.FirstDynamicState <= state);
+      Contract.Assert(curTerm != null);
+      Contract.Assert(curFactor != null);
+
+      nVars++;
+      curFactor.numVarReferences++;
+
+      if (curFactor.factorType == Automaton.StateVariableFactorStart) {
+        Contract.Assert(curFactor.variableState < 0); // not set before
+        curFactor.variableState = state;
+      } else if (curFactor.factorType == Automaton.StateExpFactorStart) {
+        curFactor.expVariableStates.Add(state);
+      } else if (curFactor.factorType == Automaton.StateLogFactorStart ||
+                 curFactor.factorType == Automaton.StateInvFactorStart) {
+        curFactor.polyVariableStates.Last().Add(state);
+      } else throw new InvalidProgramException();
+    }
+
+    public void StartNewTermInPoly() {
+      curFactor.polyVariableStates.Add(new List<int>());
+    }
+
+    public void EndFactor() {
+      // enforce non-decreasing factors 
+      if (curTerm.factors.Any() && CompareFactors(curTerm.factors.Last(), curFactor) > 0)
+        invalidExpression = true;
+      curTerm.factors.Add(curFactor);
+      curFactor = null;
+    }
+
+    public void EndTerm() {
+      // enforce non-decreasing terms (TODO: equal terms should not be allowed)
+      if (prevTerm != null && CompareTerms(prevTerm, curTerm) > 0)
+        invalidExpression = true;
+      prevTerm = curTerm;
+      curTerm = null;
+    }
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Disassembler.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Disassembler.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Disassembler.cs	(revision 14029)
@@ -0,0 +1,52 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Text;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression {
+  internal class Disassembler {
+    public static string CodeToString(byte[] code, double[] consts) {
+      var sb = new StringBuilder();
+      int pc = 0;
+      int nextParamIdx = -1;
+      while (code[pc] != (byte)OpCodes.Exit) {
+        var op = code[pc++];
+        switch (op) {
+          case (byte)OpCodes.Add: sb.Append(" + "); break;
+          case (byte)OpCodes.Mul: sb.Append(" * "); break;
+          case (byte)OpCodes.LoadConst1: sb.Append(" 1 "); break;
+          case (byte)OpCodes.LoadConst0: sb.Append(" 0 "); break;
+          case (byte)OpCodes.LoadParamN: sb.AppendFormat(" {0:N3} ", consts[++nextParamIdx]); break;
+          case (byte)OpCodes.LoadVar:
+          {
+              short arg = (short)(((short)code[pc] << 8) | (short)code[pc + 1]);
+              pc += 2;
+            sb.AppendFormat(" var{0} ", arg); break;
+          }
+          case (byte)OpCodes.Exp: sb.Append(" exp "); break;
+          case (byte)OpCodes.Log: sb.Append(" log "); break;
+          case (byte)OpCodes.Inv: sb.Append(" inv "); break;          
+        }
+      }
+      return sb.ToString();
+    }
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/ExpressionEvaluator.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/ExpressionEvaluator.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/ExpressionEvaluator.cs	(revision 14029)
@@ -0,0 +1,519 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.Contracts;
+using System.Linq;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression {
+  // evalutes expressions (on vectors)
+  internal class ExpressionEvaluator {
+    // manages it's own vector buffers
+    private readonly double[][] vectorBuffers;
+    private readonly double[][] scalarBuffers; // scalars are vectors of length 1 (to allow mixing scalars and vectors on the same stack)
+    private int lastVecBufIdx;
+    private int lastScalarBufIdx;
+
+
+    private double[] GetVectorBuffer() {
+      return vectorBuffers[--lastVecBufIdx];
+    }
+    private double[] GetScalarBuffer() {
+      return scalarBuffers[--lastScalarBufIdx];
+    }
+
+    private void ReleaseBuffer(double[] buf) {
+      if (buf.Length == 1) {
+        scalarBuffers[lastScalarBufIdx++] = buf;
+      } else {
+        vectorBuffers[lastVecBufIdx++] = buf;
+      }
+    }
+
+    public const int MaxStackSize = 100;
+    public const int MaxParams = 50;
+    private readonly int vLen;
+    private readonly double lowerEstimationLimit;
+    private readonly double upperEstimationLimit;
+    private readonly double nanReplacementValue;
+
+    private readonly double[][] stack;
+    private readonly double[][][] gradientStack;
+
+    // preallocate stack and gradient stack 
+    public ExpressionEvaluator(int vLen, double lowerEstimationLimit = double.MinValue, double upperEstimationLimit = double.MaxValue) {
+      if (vLen <= 1) throw new ArgumentException("number of elements in a variable must be > 1", "vlen");
+      this.vLen = vLen;
+      this.lowerEstimationLimit = lowerEstimationLimit;
+      this.upperEstimationLimit = upperEstimationLimit;
+      this.nanReplacementValue = (upperEstimationLimit - lowerEstimationLimit) / 2.0 + lowerEstimationLimit;
+
+      stack = new double[MaxStackSize][];
+      gradientStack = new double[MaxParams][][];
+
+      for (int k = 0; k < MaxParams; k++) {
+        gradientStack[k] = new double[MaxStackSize][];
+      }
+
+      // preallocate buffers 
+      vectorBuffers = new double[MaxStackSize * (1 + MaxParams)][];
+      scalarBuffers = new double[MaxStackSize * (1 + MaxParams)][];
+      for (int i = 0; i < MaxStackSize; i++) {
+        ReleaseBuffer(new double[vLen]);
+        ReleaseBuffer(new double[1]);
+
+        for (int k = 0; k < MaxParams; k++) {
+          ReleaseBuffer(new double[vLen]);
+          ReleaseBuffer(new double[1]);
+        }
+      }
+    }
+
+    // pred must be allocated by the caller
+    // if adjustOffsetForLogAndExp is set to true we determine c in log(c + f(x)) to make sure that c + f(x) is positive
+    public void Exec(byte[] code, double[][] vars, double[] consts, double[] pred, bool adjustOffsetForLogAndExp = false) {
+      Contract.Assert(pred != null && pred.Length >= vLen);
+      int topOfStack = -1;
+      int pc = 0;
+      int curParamIdx = -1;
+      byte op;
+      short arg;
+      // checked at the end to make sure we do not leak buffers
+      int initialScalarCount = lastScalarBufIdx;
+      int initialVectorCount = lastVecBufIdx;
+
+      while (true) {
+        ReadNext(code, ref pc, out op, out arg);
+        switch (op) {
+          case (byte)OpCodes.Nop: throw new InvalidProgramException(); // not allowed
+          case (byte)OpCodes.LoadConst0: {
+              ++topOfStack;
+              var z = GetScalarBuffer();
+              z[0] = 0;
+              stack[topOfStack] = z;
+              break;
+            }
+          case (byte)OpCodes.LoadConst1: {
+              ++topOfStack;
+              var z = GetScalarBuffer();
+              z[0] = 1.0;
+              stack[topOfStack] = z;
+              break;
+            }
+          case (byte)OpCodes.LoadParamN: {
+              ++topOfStack;
+              var c = consts[++curParamIdx];
+              var z = GetScalarBuffer();
+              z[0] = c;
+              stack[topOfStack] = z;
+              break;
+            }
+          case (byte)OpCodes.LoadVar: {
+              ++topOfStack;
+              var z = GetVectorBuffer();
+              Array.Copy(vars[arg], z, vars[arg].Length);
+              stack[topOfStack] = z;
+              break;
+            }
+          case (byte)OpCodes.Add: {
+              topOfStack--;
+              var a = stack[topOfStack + 1];
+              var b = stack[topOfStack];
+              stack[topOfStack] = Add(a, b);
+              ReleaseBuffer(a);
+              ReleaseBuffer(b);
+              break;
+            }
+          case (byte)OpCodes.Mul: {
+              topOfStack--;
+              var a = stack[topOfStack + 1];
+              var b = stack[topOfStack];
+              stack[topOfStack] = Mul(a, b);
+              ReleaseBuffer(a);
+              ReleaseBuffer(b);
+              break;
+            }
+          case (byte)OpCodes.Log: {
+              if (adjustOffsetForLogAndExp) {
+                // here we assume that the last used parameter is c in log(f(x) + c)
+                // this must match actions for producing code in the automaton!
+
+                // we can easily adjust c to make sure that f(x) + c is positive because at this point we all values for f(x)
+                var fxc = stack[topOfStack];
+                var minFx = fxc.Min() - consts[curParamIdx]; // stack[topOfStack] is f(x) + c
+
+                var delta = 1.0 - minFx - consts[curParamIdx];
+                // adjust c so that minFx + c = 1 ... log(minFx + c) = 0
+                consts[curParamIdx] += delta;
+
+                // also adjust values on stack
+                for (int i = 0; i < fxc.Length; i++) fxc[i] += delta;
+              }
+              var x = stack[topOfStack];
+              for (int i = 0; i < x.Length; i++)
+                x[i] = Math.Log(x[i]);
+              break;
+            }
+          case (byte)OpCodes.Exp: {
+              if (adjustOffsetForLogAndExp) {
+                // here we assume that the last used parameter is c in exp(f(x) * c)
+                // this must match actions for producing code in the automaton!
+
+                // adjust c to make sure that exp(f(x) * c) is not too large
+                var fxc = stack[topOfStack];
+                var maxFx = fxc.Max() / consts[curParamIdx]; // stack[topOfStack] is f(x) * c
+
+                var f = 1.0 / (maxFx * consts[curParamIdx]);
+                // adjust c so that maxFx*c = 1 TODO: this is not ideal as it enforces positive arguments to exp()
+                consts[curParamIdx] *= f;
+
+                // also adjust values on stack
+                for (int i = 0; i < fxc.Length; i++) fxc[i] *= f;
+              }
+
+              var x = stack[topOfStack];
+              for (int i = 0; i < x.Length; i++)
+                x[i] = Math.Exp(x[i]);
+              break;
+            }
+          case (byte)OpCodes.Inv: {
+              var x = stack[topOfStack];
+              for (int i = 0; i < x.Length; i++)
+                x[i] = 1.0 / (x[i]);
+              break;
+            }
+          case (byte)OpCodes.Exit:
+            Contract.Assert(topOfStack == 0);
+            var r = stack[topOfStack];
+            if (r.Length == 1) {
+              var v = double.IsNaN(r[0]) ? nanReplacementValue : Math.Min(upperEstimationLimit, Math.Max(lowerEstimationLimit, r[0]));
+              for (int i = 0; i < vLen; i++)
+                pred[i] = v;
+            } else {
+              for (int i = 0; i < vLen; i++) {
+                var v = double.IsNaN(r[i]) ? nanReplacementValue : Math.Min(upperEstimationLimit, Math.Max(lowerEstimationLimit, r[i]));
+                pred[i] = v;
+              }
+            }
+            ReleaseBuffer(r);
+            Contract.Assert(lastVecBufIdx == initialVectorCount);
+            Contract.Assert(lastScalarBufIdx == initialScalarCount);
+            return;
+        }
+      }
+    }
+
+
+    // evaluation with forward autodiff
+    // pred and gradients must be allocated by the caller
+    public void ExecGradient(byte[] code, double[][] vars, double[] consts, double[] pred, double[][] gradients) {
+      Contract.Assert(pred != null && pred.Length >= vLen);
+      int topOfStack = -1;
+      int pc = 0;
+      int curParamIdx = -1;
+      byte op;
+      short arg;
+      int nParams = consts.Length;
+      Contract.Assert(gradients != null && gradients.Length >= nParams && gradients.All(g => g.Length >= vLen));
+
+      // checked at the end to make sure we do not leak buffers
+      int initialScalarCount = lastScalarBufIdx;
+      int initialVectorCount = lastVecBufIdx;
+
+      while (true) {
+        ReadNext(code, ref pc, out op, out arg);
+        switch (op) {
+          case (byte)OpCodes.Nop: throw new InvalidProgramException(); // not allowed
+          case (byte)OpCodes.LoadConst0: {
+              ++topOfStack;
+              var z = GetScalarBuffer();
+              z[0] = 0;
+              stack[topOfStack] = z;
+              for (int k = 0; k < nParams; ++k) {
+                var b = GetScalarBuffer();
+                b[0] = 0.0;
+                gradientStack[k][topOfStack] = b;
+              }
+              break;
+            }
+          case (byte)OpCodes.LoadConst1: {
+              ++topOfStack;
+              var z = GetScalarBuffer();
+              z[0] = 1.0;
+              stack[topOfStack] = z;
+              for (int k = 0; k < nParams; ++k) {
+                var b = GetScalarBuffer();
+                b[0] = 0.0;
+                gradientStack[k][topOfStack] = b;
+              }
+              break;
+            }
+          case (byte)OpCodes.LoadParamN: {
+              var c = consts[++curParamIdx];
+              ++topOfStack;
+              var z = GetScalarBuffer();
+              z[0] = c;
+              stack[topOfStack] = z;
+              for (int k = 0; k < nParams; ++k) {
+                var b = GetScalarBuffer();
+                b[0] = k == curParamIdx ? 1.0 : 0.0;
+                gradientStack[k][topOfStack] = b;
+              }
+              break;
+            }
+          case (byte)OpCodes.LoadVar: {
+              ++topOfStack;
+              var z = GetVectorBuffer();
+              Array.Copy(vars[arg], z, vars[arg].Length);
+              stack[topOfStack] = z;
+              for (int k = 0; k < nParams; ++k) {
+                var b = GetScalarBuffer();
+                b[0] = 0.0;
+                gradientStack[k][topOfStack] = b;
+              }
+            }
+            break;
+          case (byte)OpCodes.Add: {
+              topOfStack--;
+              var a = stack[topOfStack + 1];
+              var b = stack[topOfStack];
+              stack[topOfStack] = Add(a, b);
+              ReleaseBuffer(a);
+              ReleaseBuffer(b);
+
+              // same for gradient
+              for (int k = 0; k < nParams; ++k) {
+                var ag = gradientStack[k][topOfStack + 1];
+                var bg = gradientStack[k][topOfStack];
+                gradientStack[k][topOfStack] = Add(ag, bg);
+                ReleaseBuffer(ag);
+                ReleaseBuffer(bg);
+              }
+              break;
+            }
+          case (byte)OpCodes.Mul: {
+              topOfStack--;
+              var a = stack[topOfStack + 1];
+              var b = stack[topOfStack];
+              stack[topOfStack] = Mul(a, b);
+
+              // same for gradient
+              // f(x) g(x)	f '(x) g(x) + f(x) g'(x)
+              for (int k = 0; k < nParams; ++k) {
+                var ag = gradientStack[k][topOfStack + 1];
+                var bg = gradientStack[k][topOfStack];
+                var t1 = Mul(ag, b);
+                var t2 = Mul(a, bg);
+                gradientStack[k][topOfStack] = Add(t1, t2);
+                ReleaseBuffer(ag);
+                ReleaseBuffer(bg);
+                ReleaseBuffer(t1);
+                ReleaseBuffer(t2);
+              }
+
+              ReleaseBuffer(a);
+              ReleaseBuffer(b);
+
+              break;
+            }
+          case (byte)OpCodes.Log: {
+              var x = stack[topOfStack];
+              // calc gradients first before destroying x
+              // log(f(x))' = f(x)'/f(x)
+              for (int k = 0; k < nParams; k++) {
+                var xg = gradientStack[k][topOfStack];
+                gradientStack[k][topOfStack] = Frac(xg, x);
+                ReleaseBuffer(xg);
+              }
+
+              for (int i = 0; i < x.Length; i++)
+                x[i] = Math.Log(x[i]);
+
+              break;
+            }
+          case (byte)OpCodes.Exp: {
+              var x = stack[topOfStack];
+              for (int i = 0; i < x.Length; i++)
+                x[i] = Math.Exp(x[i]);
+
+              for (int k = 0; k < nParams; k++) {
+                var xg = gradientStack[k][topOfStack];
+                gradientStack[k][topOfStack] = Mul(x, xg);  // e(f(x))' = e(f(x)) * f(x)'
+                ReleaseBuffer(xg);
+              }
+              break;
+            }
+          case (byte)OpCodes.Inv: {
+              var x = stack[topOfStack];
+              for (int i = 0; i < x.Length; i++)
+                x[i] = 1.0 / x[i];
+
+              for (int k = 0; k < nParams; k++) {
+                var xg = gradientStack[k][topOfStack];
+                // x has already been inverted above
+                // (1/f)' = -f' / f²
+                var invF = Mul(xg, x);
+                gradientStack[k][topOfStack] = Mul(invF, x, factor: -1.0);
+                ReleaseBuffer(xg);
+                ReleaseBuffer(invF);
+              }
+              break;
+            }
+          case (byte)OpCodes.Exit:
+            Contract.Assert(topOfStack == 0);
+            var r = stack[topOfStack];
+            if (r.Length == 1) {
+              var v = double.IsNaN(r[0]) ? nanReplacementValue : Math.Min(upperEstimationLimit, Math.Max(lowerEstimationLimit, r[0]));
+              for (int i = 0; i < vLen; i++)
+                pred[i] = v;
+            } else {
+              for (int i = 0; i < vLen; i++) {
+                var v = double.IsNaN(r[i]) ? nanReplacementValue : Math.Min(upperEstimationLimit, Math.Max(lowerEstimationLimit, r[i]));
+                pred[i] = v;
+              }
+            }
+            ReleaseBuffer(r);
+
+            // same for gradients
+            for (int k = 0; k < nParams; k++) {
+              var g = gradientStack[k][topOfStack];
+              if (g.Length == 1) {
+                for (int i = 0; i < vLen; i++)
+                  gradients[k][i] = g[0];
+              } else
+                Array.Copy(g, gradients[k], vLen);
+              ReleaseBuffer(g);
+            }
+
+            Contract.Assert(lastVecBufIdx == initialVectorCount);
+            Contract.Assert(lastScalarBufIdx == initialScalarCount);
+            return; // break loop
+        }
+      }
+    }
+
+    private double[] Add(double[] a, double[] b) {
+      double[] target = null;
+      if (a.Length > 1) {
+        target = GetVectorBuffer();
+        if (b.Length > 1) {
+          for (int i = 0; i < vLen; i++)
+            target[i] = a[i] + b[i];
+        } else {
+          // b == scalar
+          for (int i = 0; i < vLen; i++)
+            target[i] = a[i] + b[0];
+        }
+      } else {
+        // a == scalar
+        if (b.Length > 1) {
+          target = GetVectorBuffer();
+          for (int i = 0; i < vLen; i++)
+            target[i] = a[0] + b[i];
+        } else {
+          // b == scalar
+          target = GetScalarBuffer();
+          target[0] = a[0] + b[0];
+        }
+      }
+      return target;
+    }
+
+    private double[] Mul(double[] a, double[] b, double factor = 1.0) {
+      double[] target = null;
+      if (a.Length > 1) {
+        if (b.Length > 1) {
+          target = GetVectorBuffer();
+          for (int i = 0; i < vLen; i++)
+            target[i] = factor * a[i] * b[i];
+        } else {
+          // b == scalar
+          if (Math.Abs(b[0]) < 1E-12 /* == 0 */) {
+            target = GetScalarBuffer();
+            target[0] = 0.0;
+          } else {
+            target = GetVectorBuffer();
+            for (int i = 0; i < vLen; i++)
+              target[i] = factor * a[i] * b[0];
+          }
+        }
+      } else {
+        // a == scalar
+        if (b.Length > 1) {
+          if (Math.Abs(a[0]) < 1E-12 /* == 0 */) {
+            target = GetScalarBuffer();
+            target[0] = 0.0;
+          } else {
+            target = GetVectorBuffer();
+            for (int i = 0; i < vLen; i++)
+              target[i] = factor * a[0] * b[i];
+          }
+        } else {
+          // b == scalar
+          target = GetScalarBuffer();
+          target[0] = factor * a[0] * b[0];
+        }
+      }
+      return target;
+    }
+
+    private double[] Frac(double[] a, double[] b) {
+      double[] target = null;
+      if (a.Length > 1) {
+        target = GetVectorBuffer();
+        if (b.Length > 1) {
+          for (int i = 0; i < vLen; i++)
+            target[i] = a[i] / b[i];
+        } else {
+          // b == scalar
+          for (int i = 0; i < vLen; i++)
+            target[i] = a[i] / b[0];
+        }
+      } else {
+        // a == scalar
+        if (b.Length > 1) {
+          if (Math.Abs(a[0]) < 1E-12 /* == 0 */) {
+            target = GetScalarBuffer();
+            target[0] = 0.0;
+          } else {
+            target = GetVectorBuffer();
+            for (int i = 0; i < vLen; i++)
+              target[i] = a[0] / b[i];
+          }
+        } else {
+          // b == scalar
+          target = GetScalarBuffer();
+          target[0] = a[0] / b[0];
+        }
+      }
+      return target;
+    }
+
+    private void ReadNext(byte[] code, ref int pc, out byte op, out short s) {
+      op = code[pc++];
+      s = 0;
+      if (op == (byte)OpCodes.LoadVar) {
+        s = (short)(((short)code[pc] << 8) | (short)code[pc + 1]);
+        pc += 2;
+      }
+    }
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/MctsSymbolicRegressionAlgorithm.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/MctsSymbolicRegressionAlgorithm.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/MctsSymbolicRegressionAlgorithm.cs	(revision 14029)
@@ -0,0 +1,335 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Threading;
+using HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression.Policies;
+using HeuristicLab.Analysis;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression {
+  [Item("MCTS Symbolic Regression", "Monte carlo tree search for symbolic regression. Useful mainly as a base learner in gradient boosting.")]
+  [StorableClass]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 250)]
+  public class MctsSymbolicRegressionAlgorithm : BasicAlgorithm {
+    public override Type ProblemType {
+      get { return typeof(IRegressionProblem); }
+    }
+    public new IRegressionProblem Problem {
+      get { return (IRegressionProblem)base.Problem; }
+      set { base.Problem = value; }
+    }
+
+    #region ParameterNames
+    private const string IterationsParameterName = "Iterations";
+    private const string MaxVariablesParameterName = "Maximum variables";
+    private const string ScaleVariablesParameterName = "Scale variables";
+    private const string AllowedFactorsParameterName = "Allowed factors";
+    private const string ConstantOptimizationIterationsParameterName = "Iterations (constant optimization)";
+    private const string PolicyParameterName = "Policy";
+    private const string SeedParameterName = "Seed";
+    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+    private const string UpdateIntervalParameterName = "UpdateInterval";
+    private const string CreateSolutionParameterName = "CreateSolution";
+    private const string PunishmentFactorParameterName = "PunishmentFactor";
+
+    private const string VariableProductFactorName = "product(xi)";
+    private const string ExpFactorName = "exp(c * product(xi))";
+    private const string LogFactorName = "log(c + sum(c*product(xi))";
+    private const string InvFactorName = "1 / (1 + sum(c*product(xi))";
+    private const string FactorSumsName = "sum of multiple terms";
+    #endregion
+
+    #region ParameterProperties
+    public IFixedValueParameter<IntValue> IterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[IterationsParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> MaxVariableReferencesParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[MaxVariablesParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> ScaleVariablesParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[ScaleVariablesParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> ConstantOptimizationIterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[ConstantOptimizationIterationsParameterName]; }
+    }
+    public IValueParameter<IPolicy> PolicyParameter {
+      get { return (IValueParameter<IPolicy>)Parameters[PolicyParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> PunishmentFactorParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[PunishmentFactorParameterName]; }
+    }
+    public IValueParameter<ICheckedItemList<StringValue>> AllowedFactorsParameter {
+      get { return (IValueParameter<ICheckedItemList<StringValue>>)Parameters[AllowedFactorsParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> SeedParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+    public FixedValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (FixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> UpdateIntervalParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[UpdateIntervalParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> CreateSolutionParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
+    }
+    #endregion
+
+    #region Properties
+    public int Iterations {
+      get { return IterationsParameter.Value.Value; }
+      set { IterationsParameter.Value.Value = value; }
+    }
+    public int Seed {
+      get { return SeedParameter.Value.Value; }
+      set { SeedParameter.Value.Value = value; }
+    }
+    public bool SetSeedRandomly {
+      get { return SetSeedRandomlyParameter.Value.Value; }
+      set { SetSeedRandomlyParameter.Value.Value = value; }
+    }
+    public int MaxVariableReferences {
+      get { return MaxVariableReferencesParameter.Value.Value; }
+      set { MaxVariableReferencesParameter.Value.Value = value; }
+    }
+    public IPolicy Policy {
+      get { return PolicyParameter.Value; }
+      set { PolicyParameter.Value = value; }
+    }
+    public double PunishmentFactor {
+      get { return PunishmentFactorParameter.Value.Value; }
+      set { PunishmentFactorParameter.Value.Value = value; }
+    }
+    public ICheckedItemList<StringValue> AllowedFactors {
+      get { return AllowedFactorsParameter.Value; }
+    }
+    public int ConstantOptimizationIterations {
+      get { return ConstantOptimizationIterationsParameter.Value.Value; }
+      set { ConstantOptimizationIterationsParameter.Value.Value = value; }
+    }
+    public bool ScaleVariables {
+      get { return ScaleVariablesParameter.Value.Value; }
+      set { ScaleVariablesParameter.Value.Value = value; }
+    }
+    public bool CreateSolution {
+      get { return CreateSolutionParameter.Value.Value; }
+      set { CreateSolutionParameter.Value.Value = value; }
+    }
+    #endregion
+
+    [StorableConstructor]
+    protected MctsSymbolicRegressionAlgorithm(bool deserializing) : base(deserializing) { }
+
+    protected MctsSymbolicRegressionAlgorithm(MctsSymbolicRegressionAlgorithm original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MctsSymbolicRegressionAlgorithm(this, cloner);
+    }
+
+    public MctsSymbolicRegressionAlgorithm() {
+      Problem = new RegressionProblem(); // default problem
+
+      var defaultFactorsList = new CheckedItemList<StringValue>(
+        new string[] { VariableProductFactorName, ExpFactorName, LogFactorName, InvFactorName, FactorSumsName }
+        .Select(s => new StringValue(s).AsReadOnly())
+        ).AsReadOnly();
+      defaultFactorsList.SetItemCheckedState(defaultFactorsList.First(s => s.Value == FactorSumsName), false);
+
+      Parameters.Add(new FixedValueParameter<IntValue>(IterationsParameterName,
+        "Number of iterations", new IntValue(100000)));
+      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName,
+        "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName,
+        "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<IntValue>(MaxVariablesParameterName,
+        "Maximal number of variables references in the symbolic regression models (multiple usages of the same variable are counted)", new IntValue(5)));
+      // Parameters.Add(new FixedValueParameter<DoubleValue>(CParameterName,
+      //   "Balancing parameter in UCT formula (0 < c < 1000). Small values: greedy search. Large values: enumeration. Default: 1.0", new DoubleValue(1.0)));
+      Parameters.Add(new ValueParameter<IPolicy>(PolicyParameterName,
+        "The policy to use for selecting nodes in MCTS (e.g. Ucb)", new Ucb()));
+      PolicyParameter.Hidden = true;
+      Parameters.Add(new ValueParameter<ICheckedItemList<StringValue>>(AllowedFactorsParameterName,
+        "Choose which expressions are allowed as factors in the model.", defaultFactorsList));
+
+      Parameters.Add(new FixedValueParameter<IntValue>(ConstantOptimizationIterationsParameterName,
+        "Number of iterations for constant optimization. A small number of iterations should be sufficient for most models. " +
+        "Set to 0 to disable constants optimization.", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(ScaleVariablesParameterName,
+        "Set to true to scale all input variables to the range [0..1]", new BoolValue(false)));
+      Parameters[ScaleVariablesParameterName].Hidden = true;
+      Parameters.Add(new FixedValueParameter<DoubleValue>(PunishmentFactorParameterName, "Estimations of models can be bounded. The estimation limits are calculated in the following way (lb = mean(y) - punishmentFactor*range(y), ub = mean(y) + punishmentFactor*range(y))", new DoubleValue(10)));
+      Parameters[PunishmentFactorParameterName].Hidden = true;
+      Parameters.Add(new FixedValueParameter<IntValue>(UpdateIntervalParameterName,
+        "Number of iterations until the results are updated", new IntValue(100)));
+      Parameters[UpdateIntervalParameterName].Hidden = true;
+      Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName,
+        "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+      Parameters[CreateSolutionParameterName].Hidden = true;
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+    }
+
+    protected override void Run(CancellationToken cancellationToken) {
+      // Set up the algorithm
+      if (SetSeedRandomly) Seed = new System.Random().Next();
+
+      // Set up the results display
+      var iterations = new IntValue(0);
+      Results.Add(new Result("Iterations", iterations));
+
+      var bestSolutionIteration = new IntValue(0);
+      Results.Add(new Result("Best solution iteration", bestSolutionIteration));
+
+      var table = new DataTable("Qualities");
+      table.Rows.Add(new DataRow("Best quality"));
+      table.Rows.Add(new DataRow("Current best quality"));
+      table.Rows.Add(new DataRow("Average quality"));
+      Results.Add(new Result("Qualities", table));
+
+      var bestQuality = new DoubleValue();
+      Results.Add(new Result("Best quality", bestQuality));
+
+      var curQuality = new DoubleValue();
+      Results.Add(new Result("Current best quality", curQuality));
+
+      var avgQuality = new DoubleValue();
+      Results.Add(new Result("Average quality", avgQuality));
+
+      var totalRollouts = new IntValue();
+      Results.Add(new Result("Total rollouts", totalRollouts));
+      var effRollouts = new IntValue();
+      Results.Add(new Result("Effective rollouts", effRollouts));
+      var funcEvals = new IntValue();
+      Results.Add(new Result("Function evaluations", funcEvals));
+      var gradEvals = new IntValue();
+      Results.Add(new Result("Gradient evaluations", gradEvals));
+
+
+      // same as in SymbolicRegressionSingleObjectiveProblem
+      var y = Problem.ProblemData.Dataset.GetDoubleValues(Problem.ProblemData.TargetVariable,
+        Problem.ProblemData.TrainingIndices);
+      var avgY = y.Average();
+      var minY = y.Min();
+      var maxY = y.Max();
+      var range = maxY - minY;
+      var lowerLimit = avgY - PunishmentFactor * range;
+      var upperLimit = avgY + PunishmentFactor * range;
+
+      // init
+      var problemData = (IRegressionProblemData)Problem.ProblemData.Clone();
+      if (!AllowedFactors.CheckedItems.Any()) throw new ArgumentException("At least on type of factor must be allowed");
+      var state = MctsSymbolicRegressionStatic.CreateState(problemData, (uint)Seed, MaxVariableReferences, ScaleVariables, ConstantOptimizationIterations,
+        Policy,
+        lowerLimit, upperLimit,
+        allowProdOfVars: AllowedFactors.CheckedItems.Any(s => s.Value.Value == VariableProductFactorName),
+        allowExp: AllowedFactors.CheckedItems.Any(s => s.Value.Value == ExpFactorName),
+        allowLog: AllowedFactors.CheckedItems.Any(s => s.Value.Value == LogFactorName),
+        allowInv: AllowedFactors.CheckedItems.Any(s => s.Value.Value == InvFactorName),
+        allowMultipleTerms: AllowedFactors.CheckedItems.Any(s => s.Value.Value == FactorSumsName)
+        );
+
+      var updateInterval = UpdateIntervalParameter.Value.Value;
+      double sumQ = 0.0;
+      double bestQ = 0.0;
+      double curBestQ = 0.0;
+      int n = 0;
+      // Loop until iteration limit reached or canceled.
+      for (int i = 0; i < Iterations && !state.Done; i++) {
+        cancellationToken.ThrowIfCancellationRequested();
+
+        var q = MctsSymbolicRegressionStatic.MakeStep(state);
+        sumQ += q; // sum of qs in the last updateinterval iterations
+        curBestQ = Math.Max(q, curBestQ); // the best q in the last updateinterval iterations
+        bestQ = Math.Max(q, bestQ); // the best q overall
+        n++;
+        // iteration results
+        if (n == updateInterval) {
+          if (bestQ > bestQuality.Value) {
+            bestSolutionIteration.Value = i;
+          }
+          bestQuality.Value = bestQ;
+          curQuality.Value = curBestQ;
+          avgQuality.Value = sumQ / n;
+          sumQ = 0.0;
+          curBestQ = 0.0;
+
+          funcEvals.Value = state.FuncEvaluations;
+          gradEvals.Value = state.GradEvaluations;
+          effRollouts.Value = state.EffectiveRollouts;
+          totalRollouts.Value = state.TotalRollouts;
+
+          table.Rows["Best quality"].Values.Add(bestQuality.Value);
+          table.Rows["Current best quality"].Values.Add(curQuality.Value);
+          table.Rows["Average quality"].Values.Add(avgQuality.Value);
+          iterations.Value += n;
+          n = 0;
+        }
+      }
+
+      // final results
+      if (n > 0) {
+        if (bestQ > bestQuality.Value) {
+          bestSolutionIteration.Value = iterations.Value + n;
+        }
+        bestQuality.Value = bestQ;
+        curQuality.Value = curBestQ;
+        avgQuality.Value = sumQ / n;
+
+        funcEvals.Value = state.FuncEvaluations;
+        gradEvals.Value = state.GradEvaluations;
+        effRollouts.Value = state.EffectiveRollouts;
+        totalRollouts.Value = state.TotalRollouts;
+
+        table.Rows["Best quality"].Values.Add(bestQuality.Value);
+        table.Rows["Current best quality"].Values.Add(curQuality.Value);
+        table.Rows["Average quality"].Values.Add(avgQuality.Value);
+        iterations.Value = iterations.Value + n;
+
+      }
+
+
+      Results.Add(new Result("Best solution quality (train)", new DoubleValue(state.BestSolutionTrainingQuality)));
+      Results.Add(new Result("Best solution quality (test)", new DoubleValue(state.BestSolutionTestQuality)));
+
+
+      // produce solution 
+      if (CreateSolution) {
+        var model = state.BestModel;
+
+        // otherwise we produce a regression solution
+        Results.Add(new Result("Solution", model.CreateRegressionSolution(problemData)));
+      }
+    }
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/MctsSymbolicRegressionStatic.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/MctsSymbolicRegressionStatic.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/MctsSymbolicRegressionStatic.cs	(revision 14029)
@@ -0,0 +1,473 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.Contracts;
+using System.Linq;
+using HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression.Policies;
+using HeuristicLab.Core;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
+using HeuristicLab.Random;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression {
+  public static class MctsSymbolicRegressionStatic {
+    // TODO: SGD with adagrad instead of lbfgs?
+    // TODO: check Taylor expansion capabilities (ln(x), sqrt(x), exp(x)) in combination with GBT
+    // TODO: optimize for 3 targets concurrently (y, 1/y, exp(y), and log(y))? Would simplify the number of possible expressions again
+    #region static API
+
+    public interface IState {
+      bool Done { get; }
+      ISymbolicRegressionModel BestModel { get; }
+      double BestSolutionTrainingQuality { get; }
+      double BestSolutionTestQuality { get; }
+      int TotalRollouts { get; }
+      int EffectiveRollouts { get; }
+      int FuncEvaluations { get; }
+      int GradEvaluations { get; } // number of gradient evaluations (* num parameters) to get a value representative of the effort comparable to the number of function evaluations
+      // TODO other stats on LM optimizer might be interesting here
+    }
+
+    // created through factory method
+    private class State : IState {
+      private const int MaxParams = 100;
+
+      // state variables used by MCTS
+      internal readonly Automaton automaton;
+      internal IRandom random { get; private set; }
+      internal readonly Tree tree;
+      internal readonly Func<byte[], int, double> evalFun;
+      internal readonly IPolicy treePolicy;
+      // MCTS might get stuck. Track statistics on the number of effective rollouts
+      internal int totalRollouts;
+      internal int effectiveRollouts;
+
+
+      // state variables used only internally (for eval function)
+      private readonly IRegressionProblemData problemData;
+      private readonly double[][] x;
+      private readonly double[] y;
+      private readonly double[][] testX;
+      private readonly double[] testY;
+      private readonly double[] scalingFactor;
+      private readonly double[] scalingOffset;
+      private readonly int constOptIterations;
+      private readonly double lowerEstimationLimit, upperEstimationLimit;
+
+      private readonly ExpressionEvaluator evaluator, testEvaluator;
+
+      // values for best solution
+      private double bestRSq;
+      private byte[] bestCode;
+      private int bestNParams;
+      private double[] bestConsts;
+
+      // stats
+      private int funcEvaluations;
+      private int gradEvaluations;
+
+      // buffers
+      private readonly double[] ones; // vector of ones (as default params)
+      private readonly double[] constsBuf;
+      private readonly double[] predBuf, testPredBuf;
+      private readonly double[][] gradBuf;
+
+      public State(IRegressionProblemData problemData, uint randSeed, int maxVariables, bool scaleVariables, int constOptIterations,
+        IPolicy treePolicy = null,
+        double lowerEstimationLimit = double.MinValue, double upperEstimationLimit = double.MaxValue,
+        bool allowProdOfVars = true,
+        bool allowExp = true,
+        bool allowLog = true,
+        bool allowInv = true,
+        bool allowMultipleTerms = false) {
+
+        this.problemData = problemData;
+        this.constOptIterations = constOptIterations;
+        this.evalFun = this.Eval;
+        this.lowerEstimationLimit = lowerEstimationLimit;
+        this.upperEstimationLimit = upperEstimationLimit;
+
+        random = new MersenneTwister(randSeed);
+
+        // prepare data for evaluation
+        double[][] x;
+        double[] y;
+        double[][] testX;
+        double[] testY;
+        double[] scalingFactor;
+        double[] scalingOffset;
+        // get training and test datasets (scale linearly based on training set if required)
+        GenerateData(problemData, scaleVariables, problemData.TrainingIndices, out x, out y, out scalingFactor, out scalingOffset);
+        GenerateData(problemData, problemData.TestIndices, scalingFactor, scalingOffset, out testX, out testY);
+        this.x = x;
+        this.y = y;
+        this.testX = testX;
+        this.testY = testY;
+        this.scalingFactor = scalingFactor;
+        this.scalingOffset = scalingOffset;
+        this.evaluator = new ExpressionEvaluator(y.Length, lowerEstimationLimit, upperEstimationLimit);
+        // we need a separate evaluator because the vector length for the test dataset might differ 
+        this.testEvaluator = new ExpressionEvaluator(testY.Length, lowerEstimationLimit, upperEstimationLimit);
+
+        this.automaton = new Automaton(x, maxVariables, allowProdOfVars, allowExp, allowLog, allowInv, allowMultipleTerms);
+        this.treePolicy = treePolicy ?? new Ucb();
+        this.tree = new Tree() { state = automaton.CurrentState, actionStatistics = treePolicy.CreateActionStatistics() };
+
+        // reset best solution
+        this.bestRSq = 0;
+        // code for default solution (constant model)
+        this.bestCode = new byte[] { (byte)OpCodes.LoadConst0, (byte)OpCodes.Exit };
+        this.bestNParams = 0;
+        this.bestConsts = null;
+
+        // init buffers
+        this.ones = Enumerable.Repeat(1.0, MaxParams).ToArray();
+        constsBuf = new double[MaxParams];
+        this.predBuf = new double[y.Length];
+        this.testPredBuf = new double[testY.Length];
+
+        this.gradBuf = Enumerable.Range(0, MaxParams).Select(_ => new double[y.Length]).ToArray();
+      }
+
+      #region IState inferface
+      public bool Done { get { return tree != null && tree.Done; } }
+
+      public double BestSolutionTrainingQuality {
+        get {
+          evaluator.Exec(bestCode, x, bestConsts, predBuf);
+          return RSq(y, predBuf);
+        }
+      }
+
+      public double BestSolutionTestQuality {
+        get {
+          testEvaluator.Exec(bestCode, testX, bestConsts, testPredBuf);
+          return RSq(testY, testPredBuf);
+        }
+      }
+
+      // takes the code of the best solution and creates and equivalent symbolic regression model
+      public ISymbolicRegressionModel BestModel {
+        get {
+          var treeGen = new SymbolicExpressionTreeGenerator(problemData.AllowedInputVariables.ToArray());
+          var interpreter = new SymbolicDataAnalysisExpressionTreeLinearInterpreter();
+
+          var t = new SymbolicExpressionTree(treeGen.Exec(bestCode, bestConsts, bestNParams, scalingFactor, scalingOffset));
+          var model = new SymbolicRegressionModel(problemData.TargetVariable, t, interpreter, lowerEstimationLimit, upperEstimationLimit);
+
+          // model has already been scaled linearly in Eval
+          return model;
+        }
+      }
+
+      public int TotalRollouts { get { return totalRollouts; } }
+      public int EffectiveRollouts { get { return effectiveRollouts; } }
+      public int FuncEvaluations { get { return funcEvaluations; } }
+      public int GradEvaluations { get { return gradEvaluations; } } // number of gradient evaluations (* num parameters) to get a value representative of the effort comparable to the number of function evaluations
+
+      #endregion
+
+      private double Eval(byte[] code, int nParams) {
+        double[] optConsts;
+        double q;
+        Eval(code, nParams, out q, out optConsts);
+
+        if (q > bestRSq) {
+          bestRSq = q;
+          bestNParams = nParams;
+          this.bestCode = new byte[code.Length];
+          this.bestConsts = new double[bestNParams];
+
+          Array.Copy(code, bestCode, code.Length);
+          Array.Copy(optConsts, bestConsts, bestNParams);
+        }
+
+        return q;
+      }
+
+      private void Eval(byte[] code, int nParams, out double rsq, out double[] optConsts) {
+        // we make a first pass to determine a valid starting configuration for all constants
+        // constant c in log(c + f(x)) is adjusted to guarantee that x is positive (see expression evaluator)
+        // scale and offset are set to optimal starting configuration
+        // assumes scale is the first param and offset is the last param
+        double alpha;
+        double beta;
+
+        // reset constants
+        Array.Copy(ones, constsBuf, nParams);
+        evaluator.Exec(code, x, constsBuf, predBuf, adjustOffsetForLogAndExp: true);
+        funcEvaluations++;
+
+        // calc opt scaling (alpha*f(x) + beta)
+        OnlineCalculatorError error;
+        OnlineLinearScalingParameterCalculator.Calculate(predBuf, y, out alpha, out beta, out error);
+        if (error == OnlineCalculatorError.None) {
+          constsBuf[0] *= beta;
+          constsBuf[nParams - 1] = constsBuf[nParams - 1] * beta + alpha;
+        }
+        if (nParams <= 2 || constOptIterations <= 0) {
+          // if we don't need to optimize parameters then we are done
+          // changing scale and offset does not influence r²
+          rsq = RSq(y, predBuf);
+          optConsts = constsBuf;
+        } else {
+          // optimize constants using the starting point calculated above
+          OptimizeConstsLm(code, constsBuf, nParams, 0.0, nIters: constOptIterations);
+
+          evaluator.Exec(code, x, constsBuf, predBuf);
+          funcEvaluations++;
+
+          rsq = RSq(y, predBuf);
+          optConsts = constsBuf;
+        }
+      }
+
+
+
+      #region helpers
+      private static double RSq(IEnumerable<double> x, IEnumerable<double> y) {
+        OnlineCalculatorError error;
+        double r = OnlinePearsonsRCalculator.Calculate(x, y, out error);
+        return error == OnlineCalculatorError.None ? r * r : 0.0;
+      }
+
+
+      private void OptimizeConstsLm(byte[] code, double[] consts, int nParams, double epsF = 0.0, int nIters = 100) {
+        double[] optConsts = new double[nParams]; // allocate a smaller buffer for constants opt (TODO perf?)
+        Array.Copy(consts, optConsts, nParams);
+
+        alglib.minlmstate state;
+        alglib.minlmreport rep = null;
+        alglib.minlmcreatevj(y.Length, optConsts, out state);
+        alglib.minlmsetcond(state, 0.0, epsF, 0.0, nIters);
+        //alglib.minlmsetgradientcheck(state, 0.000001);
+        alglib.minlmoptimize(state, Func, FuncAndJacobian, null, code);
+        alglib.minlmresults(state, out optConsts, out rep);
+        funcEvaluations += rep.nfunc;
+        gradEvaluations += rep.njac * nParams;
+
+        if (rep.terminationtype < 0) throw new ArgumentException("lm failed: termination type = " + rep.terminationtype);
+
+        // only use optimized constants if successful
+        if (rep.terminationtype >= 0) {
+          Array.Copy(optConsts, consts, optConsts.Length);
+        }
+      }
+
+      private void Func(double[] arg, double[] fi, object obj) {
+        var code = (byte[])obj;
+        evaluator.Exec(code, x, arg, predBuf); // gradients are nParams x vLen
+        for (int r = 0; r < predBuf.Length; r++) {
+          var res = predBuf[r] - y[r];
+          fi[r] = res;
+        }
+      }
+      private void FuncAndJacobian(double[] arg, double[] fi, double[,] jac, object obj) {
+        int nParams = arg.Length;
+        var code = (byte[])obj;
+        evaluator.ExecGradient(code, x, arg, predBuf, gradBuf); // gradients are nParams x vLen
+        for (int r = 0; r < predBuf.Length; r++) {
+          var res = predBuf[r] - y[r];
+          fi[r] = res;
+
+          for (int k = 0; k < nParams; k++) {
+            jac[r, k] = gradBuf[k][r];
+          }
+        }
+      }
+      #endregion
+    }
+
+    public static IState CreateState(IRegressionProblemData problemData, uint randSeed, int maxVariables = 3,
+      bool scaleVariables = true, int constOptIterations = 0,
+      IPolicy policy = null,
+      double lowerEstimationLimit = double.MinValue, double upperEstimationLimit = double.MaxValue,
+      bool allowProdOfVars = true,
+      bool allowExp = true,
+      bool allowLog = true,
+      bool allowInv = true,
+      bool allowMultipleTerms = false
+      ) {
+      return new State(problemData, randSeed, maxVariables, scaleVariables, constOptIterations,
+        policy,
+        lowerEstimationLimit, upperEstimationLimit,
+        allowProdOfVars, allowExp, allowLog, allowInv, allowMultipleTerms);
+    }
+
+    // returns the quality of the evaluated solution
+    public static double MakeStep(IState state) {
+      var mctsState = state as State;
+      if (mctsState == null) throw new ArgumentException("state");
+      if (mctsState.Done) throw new NotSupportedException("The tree search has enumerated all possible solutions.");
+
+      return TreeSearch(mctsState);
+    }
+    #endregion
+
+    private static double TreeSearch(State mctsState) {
+      var automaton = mctsState.automaton;
+      var tree = mctsState.tree;
+      var eval = mctsState.evalFun;
+      var rand = mctsState.random;
+      var treePolicy = mctsState.treePolicy;
+      double q = 0;
+      bool success = false;
+      do {
+        automaton.Reset();
+        success = TryTreeSearchRec(rand, tree, automaton, eval, treePolicy, out q);
+        mctsState.totalRollouts++;
+      } while (!success && !tree.Done);
+      mctsState.effectiveRollouts++;
+      return q;
+    }
+
+    // tree search might fail because of constraints for expressions
+    // in this case we get stuck we just restart
+    // see ConstraintHandler.cs for more info
+    private static bool TryTreeSearchRec(IRandom rand, Tree tree, Automaton automaton, Func<byte[], int, double> eval, IPolicy treePolicy,
+      out double q) {
+      Tree selectedChild = null;
+      Contract.Assert(tree.state == automaton.CurrentState);
+      Contract.Assert(!tree.Done);
+      if (tree.children == null) {
+        if (automaton.IsFinalState(tree.state)) {
+          // final state
+          tree.Done = true;
+
+          // EVALUATE
+          byte[] code; int nParams;
+          automaton.GetCode(out code, out nParams);
+          q = eval(code, nParams);
+
+          treePolicy.Update(tree.actionStatistics, q);
+          return true; // we reached a final state
+        } else {
+          // EXPAND 
+          int[] possibleFollowStates;
+          int nFs;
+          automaton.FollowStates(automaton.CurrentState, out possibleFollowStates, out nFs);
+          if (nFs == 0) {
+            // stuck in a dead end (no final state and no allowed follow states)
+            q = 0;
+            tree.Done = true;
+            tree.children = null;
+            return false;
+          }
+          tree.children = new Tree[nFs];
+          for (int i = 0; i < tree.children.Length; i++)
+            tree.children[i] = new Tree() { children = null, state = possibleFollowStates[i], actionStatistics = treePolicy.CreateActionStatistics() };
+
+          selectedChild = nFs > 1 ? SelectFinalOrRandom(automaton, tree, rand) : tree.children[0];
+        }
+      } else {
+        // tree.children != null
+        // UCT selection within tree
+        int selectedIdx = 0;
+        if (tree.children.Length > 1) {
+          selectedIdx = treePolicy.Select(tree.children.Select(ch => ch.actionStatistics), rand);
+        }
+        selectedChild = tree.children[selectedIdx];
+      }
+      // make selected step and recurse
+      automaton.Goto(selectedChild.state);
+      var success = TryTreeSearchRec(rand, selectedChild, automaton, eval, treePolicy, out q);
+      if (success) {
+        // only update if successful
+        treePolicy.Update(tree.actionStatistics, q);
+      }
+
+      tree.Done = tree.children.All(ch => ch.Done);
+      if (tree.Done) {
+        tree.children = null; // cut off the sub-branch if it has been fully explored
+      }
+      return success;
+    }
+
+    private static Tree SelectFinalOrRandom(Automaton automaton, Tree tree, IRandom rand) {
+      // if one of the new children leads to a final state then go there
+      // otherwise choose a random child
+      int selectedChildIdx = -1;
+      // find first final state if there is one
+      for (int i = 0; i < tree.children.Length; i++) {
+        if (automaton.IsFinalState(tree.children[i].state)) {
+          selectedChildIdx = i;
+          break;
+        }
+      }
+      // no final state -> select a the first child
+      if (selectedChildIdx == -1) {
+        selectedChildIdx = 0;
+      }
+      return tree.children[selectedChildIdx];
+    }
+
+    // scales data and extracts values from dataset into arrays
+    private static void GenerateData(IRegressionProblemData problemData, bool scaleVariables, IEnumerable<int> rows,
+      out double[][] xs, out double[] y, out double[] scalingFactor, out double[] scalingOffset) {
+      xs = new double[problemData.AllowedInputVariables.Count()][];
+
+      var i = 0;
+      if (scaleVariables) {
+        scalingFactor = new double[xs.Length];
+        scalingOffset = new double[xs.Length];
+      } else {
+        scalingFactor = null;
+        scalingOffset = null;
+      }
+      foreach (var var in problemData.AllowedInputVariables) {
+        if (scaleVariables) {
+          var minX = problemData.Dataset.GetDoubleValues(var, rows).Min();
+          var maxX = problemData.Dataset.GetDoubleValues(var, rows).Max();
+          var range = maxX - minX;
+
+          // scaledX = (x - min) / range
+          var sf = 1.0 / range;
+          var offset = -minX / range;
+          scalingFactor[i] = sf;
+          scalingOffset[i] = offset;
+          i++;
+        }
+      }
+
+      GenerateData(problemData, rows, scalingFactor, scalingOffset, out xs, out y);
+    }
+
+    // extract values from dataset into arrays
+    private static void GenerateData(IRegressionProblemData problemData, IEnumerable<int> rows, double[] scalingFactor, double[] scalingOffset,
+     out double[][] xs, out double[] y) {
+      xs = new double[problemData.AllowedInputVariables.Count()][];
+
+      int i = 0;
+      foreach (var var in problemData.AllowedInputVariables) {
+        var sf = scalingFactor == null ? 1.0 : scalingFactor[i];
+        var offset = scalingFactor == null ? 0.0 : scalingOffset[i];
+        xs[i++] =
+          problemData.Dataset.GetDoubleValues(var, rows).Select(xi => xi * sf + offset).ToArray();
+      }
+
+      y = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows).ToArray();
+    }
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/OpCodes.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/OpCodes.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/OpCodes.cs	(revision 14029)
@@ -0,0 +1,36 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression {
+  internal enum OpCodes : byte {
+    Nop,
+    LoadConst0,
+    LoadConst1, // load the constant 1
+    LoadParamN, // load next parameter 
+    LoadVar,    // + 2 byte parameters
+    Mul,
+    Add,
+    Log,
+    Exp,
+    Inv,
+    Exit
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Policies/EpsGreedy.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Policies/EpsGreedy.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Policies/EpsGreedy.cs	(revision 14029)
@@ -0,0 +1,93 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics.Contracts;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression.Policies {
+  [StorableClass]
+  [Item("EpsilonGreedy", "Epsilon greedy policy with parameter eps to balance between exploitation and exploration")]
+  public class EpsilonGreedy : PolicyBase {
+    private class ActionStatistics : IActionStatistics {
+      public double SumQuality { get; set; }
+      public double AverageQuality { get { return SumQuality / Tries; } }
+      public int Tries { get; set; }
+      public bool Done { get; set; }
+    }
+    private List<int> buf = new List<int>();
+
+    public IFixedValueParameter<DoubleValue> EpsParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters["Eps"]; }
+    }
+
+    public double Eps {
+      get { return EpsParameter.Value.Value; }
+      set { EpsParameter.Value.Value = value; }
+    }
+
+    [StorableConstructor]
+    protected EpsilonGreedy(bool deserializing) : base(deserializing) { }
+    protected EpsilonGreedy(EpsilonGreedy original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public EpsilonGreedy()
+      : base() {
+      Parameters.Add(new FixedValueParameter<DoubleValue>("Eps", "Rate of random selection 0 (greedy) <= eps <= 1 (random)", new DoubleValue(0.1)));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new EpsilonGreedy(this, cloner);
+    }
+
+    public override int Select(IEnumerable<IActionStatistics> actions, IRandom random) {
+      return Select(actions, random, Eps, buf);
+    }
+
+    public override void Update(IActionStatistics action, double q) {
+      var a = action as ActionStatistics;
+      a.SumQuality += q;
+      a.Tries++;
+    }
+
+    public override IActionStatistics CreateActionStatistics() {
+      return new ActionStatistics();
+    }
+
+    private static int Select(IEnumerable<IActionStatistics> actions, IRandom rand, double c, IList<int> buf) {
+      buf.Clear();
+      if (rand.NextDouble() >= c) {
+        // select best
+        var bestQ = double.NegativeInfinity;
+        int aIdx = -1;
+        foreach (var a in actions) {
+          ++aIdx;
+          if (a.Done) continue;
+          var actionQ = a.Tries > 0 ? a.AverageQuality : double.PositiveInfinity; // always try unvisited actions first
+          if (actionQ > bestQ) {
+            buf.Clear();
+            buf.Add(aIdx);
+            bestQ = actionQ;
+          } else if (actionQ >= bestQ) {
+            buf.Add(aIdx);
+          }
+        }
+        return buf[rand.Next(buf.Count)];
+      } else {
+        // random selection
+        int aIdx = -1;
+        foreach (var a in actions) {
+          ++aIdx;
+          if (a.Done) continue;
+          buf.Add(aIdx);
+        }
+        return buf[rand.Next(buf.Count)];
+      }
+    }
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Policies/IActionStatistics.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Policies/IActionStatistics.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Policies/IActionStatistics.cs	(revision 14029)
@@ -0,0 +1,13 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression.Policies {
+  public interface IActionStatistics {
+    double AverageQuality { get; }
+    int Tries { get; }
+    bool Done { get; set; }
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Policies/IPolicy.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Policies/IPolicy.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Policies/IPolicy.cs	(revision 14029)
@@ -0,0 +1,14 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Core;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression.Policies {
+  public interface IPolicy : IItem {
+    int Select(IEnumerable<IActionStatistics> actions, IRandom random);
+    void Update(IActionStatistics action, double q);
+    IActionStatistics CreateActionStatistics();
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Policies/PolicyBase.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Policies/PolicyBase.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Policies/PolicyBase.cs	(revision 14029)
@@ -0,0 +1,36 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics.Contracts;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression.Policies {
+  [StorableClass]
+  public abstract class PolicyBase : Item, IParameterizedItem, IPolicy {
+    [Storable]
+    public IKeyedItemCollection<string, IParameter> Parameters { get; private set; }
+
+    [StorableConstructor]
+    protected PolicyBase(bool deserializing) : base(deserializing) { }
+    protected PolicyBase(PolicyBase original, Cloner cloner)
+      : base(original, cloner) {
+      Parameters = cloner.Clone(original.Parameters);
+    }
+    protected PolicyBase()
+      : base() {
+      Parameters = new ParameterCollection();
+    }
+
+    public void CollectParameterValues(IDictionary<string, IItem> values) {
+      foreach (var p in Parameters) values.Add(this.ItemName + "." + p.Name, p.ActualValue);
+    }
+
+    public abstract int Select(IEnumerable<IActionStatistics> actions, IRandom random);
+    public abstract void Update(IActionStatistics action, double q);
+    public abstract IActionStatistics CreateActionStatistics();
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Policies/Ucb.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Policies/Ucb.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Policies/Ucb.cs	(revision 14029)
@@ -0,0 +1,97 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics.Contracts;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression.Policies {
+  [StorableClass]
+  [Item("Ucb Policy", "Ucb with parameter c to balance between exploitation and exploration")]
+  public class Ucb : PolicyBase {
+    private class ActionStatistics : IActionStatistics {
+      public double SumQuality { get; set; }
+      public double AverageQuality { get { return SumQuality / Tries; } }
+      public int Tries { get; set; }
+      public bool Done { get; set; }
+    }
+    private List<int> buf = new List<int>();
+
+    public IFixedValueParameter<DoubleValue> CParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters["C"]; }
+    }
+
+    public double C {
+      get { return CParameter.Value.Value; }
+      set { CParameter.Value.Value = value; }
+    }
+
+    [StorableConstructor]
+    protected Ucb(bool deserializing) : base(deserializing) { }
+    protected Ucb(Ucb original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public Ucb()
+      : base() {
+      Parameters.Add(new FixedValueParameter<DoubleValue>("C", "Parameter to balance between exploration and exploitation 0 <= c < 100", new DoubleValue(Math.Sqrt(2))));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new Ucb(this, cloner);
+    }
+
+    public override int Select(IEnumerable<IActionStatistics> actions, IRandom random) {
+      return Select(actions, random, C, buf);
+    }
+
+    public override void Update(IActionStatistics action, double q) {
+      var a = action as ActionStatistics;
+      a.SumQuality += q;
+      a.Tries++;
+    }
+
+    public override IActionStatistics CreateActionStatistics() {
+      return new ActionStatistics();
+    }
+
+    private static int Select(IEnumerable<IActionStatistics> actions, IRandom rand, double c, IList<int> buf) {
+      // determine total tries of still active actions
+      int totalTries = 0;
+      buf.Clear();
+      int aIdx = -1;
+      foreach (var a in actions) {
+        ++aIdx;
+        if (a.Done) continue;
+        if (a.Tries == 0) buf.Add(aIdx);
+        else totalTries += a.Tries;
+      }
+      // if there are unvisited actions select a random action
+      if (buf.Any()) {
+        return buf[rand.Next(buf.Count)];
+      }
+      Contract.Assert(totalTries > 0);
+      double logTotalTries = Math.Log(totalTries);
+      var bestQ = double.NegativeInfinity;
+      aIdx = -1;
+      foreach (var a in actions) {
+        ++aIdx;
+        if (a.Done) continue;
+        var actionQ = a.AverageQuality + c * Math.Sqrt(logTotalTries / a.Tries);
+        if (actionQ > bestQ) {
+          buf.Clear();
+          buf.Add(aIdx);
+          bestQ = actionQ;
+        } else if (actionQ >= bestQ) {
+          buf.Add(aIdx);
+        }
+      }
+      return buf[rand.Next(buf.Count)];
+    }
+
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Policies/UcbTuned.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Policies/UcbTuned.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Policies/UcbTuned.cs	(revision 14029)
@@ -0,0 +1,101 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics.Contracts;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression.Policies {
+  [StorableClass]
+  [Item("UcbTuned Policy", "UcbTuned is similar to Ucb but tracks empirical variance. Use parameter c to balance between exploitation and exploration")]
+  public class UcbTuned : PolicyBase {
+    private class ActionStatistics : IActionStatistics {
+      public double SumQuality { get; set; }
+      public double SumSqrQuality { get; set; }
+      public double AverageQuality { get { return SumQuality / Tries; } }
+      public double QualityVariance { get { return SumSqrQuality / Tries - AverageQuality * AverageQuality; } }
+      public int Tries { get; set; }
+      public bool Done { get; set; }
+    }
+    private List<int> buf = new List<int>();
+
+    public IFixedValueParameter<DoubleValue> CParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters["C"]; }
+    }
+
+    public double C {
+      get { return CParameter.Value.Value; }
+      set { CParameter.Value.Value = value; }
+    }
+
+    [StorableConstructor]
+    protected UcbTuned(bool deserializing) : base(deserializing) { }
+    protected UcbTuned(UcbTuned original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public UcbTuned()
+      : base() {
+      Parameters.Add(new FixedValueParameter<DoubleValue>("C", "Parameter to balance between exploration and exploitation 0 <= c < 100", new DoubleValue(Math.Sqrt(2))));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new UcbTuned(this, cloner);
+    }
+
+    public override int Select(IEnumerable<IActionStatistics> actions, IRandom random) {
+      return Select(actions, random, C, buf);
+    }
+
+    public override void Update(IActionStatistics action, double q) {
+      var a = action as ActionStatistics;
+      a.SumQuality += q;
+      a.SumSqrQuality += q * q;
+      a.Tries++;
+    }
+
+    public override IActionStatistics CreateActionStatistics() {
+      return new ActionStatistics();
+    }
+
+    private static int Select(IEnumerable<IActionStatistics> actions, IRandom rand, double c, IList<int> buf) {
+      // determine total tries of still active actions
+      int totalTries = 0;
+      buf.Clear();
+      int aIdx = -1;
+      foreach (var a in actions) {
+        ++aIdx;
+        if (a.Done) continue;
+        if (a.Tries == 0) buf.Add(aIdx);
+        else totalTries += a.Tries;
+      }
+      // if there are unvisited actions select a random action
+      if (buf.Any()) {
+        return buf[rand.Next(buf.Count)];
+      }
+      Contract.Assert(totalTries > 0);
+      double logTotalTries = Math.Log(totalTries);
+      var bestQ = double.NegativeInfinity;
+      aIdx = -1;
+      foreach (var a in actions.Cast<ActionStatistics>()) {
+        ++aIdx;
+        if (a.Done) continue;
+        var varianceBound = a.QualityVariance + Math.Sqrt(2.0 * logTotalTries / a.Tries);
+        if (varianceBound > 0.25) varianceBound = 0.25;
+        var actionQ = a.AverageQuality + c * Math.Sqrt(logTotalTries / a.Tries * varianceBound);
+        if (actionQ > bestQ) {
+          buf.Clear();
+          buf.Add(aIdx);
+          bestQ = actionQ;
+        } else if (actionQ >= bestQ) {
+          buf.Add(aIdx);
+        }
+      }
+      return buf[rand.Next(buf.Count)];
+    }
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/SymbolicExpressionGenerator.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/SymbolicExpressionGenerator.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/SymbolicExpressionGenerator.cs	(revision 14029)
@@ -0,0 +1,183 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+using System;
+using System.Diagnostics.Contracts;
+using System.Linq;
+
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression {
+
+  // translates byte code to a symbolic expression tree
+  internal class SymbolicExpressionTreeGenerator {
+    const int MaxStackSize = 100;
+    private readonly ISymbolicExpressionTreeNode[] stack;
+    private readonly ConstantTreeNode const0;
+    private readonly ConstantTreeNode const1;
+    private readonly Addition addSy;
+    private readonly Multiplication mulSy;
+    private readonly Exponential expSy;
+    private readonly Logarithm logSy;
+    private readonly Division divSy;
+    private readonly VariableTreeNode varNode;
+    private readonly string[] variableNames;
+    private readonly StartSymbol startSy;
+    private readonly ProgramRootSymbol progRootSy;
+
+    public SymbolicExpressionTreeGenerator(string[] variableNames) {
+      stack = new ISymbolicExpressionTreeNode[MaxStackSize];
+      var grammar = new TypeCoherentExpressionGrammar();
+      this.variableNames = variableNames;
+
+      grammar.ConfigureAsDefaultRegressionGrammar();
+      const0 = (ConstantTreeNode)grammar.Symbols.OfType<Constant>().First().CreateTreeNode();
+      const0.Value = 0;
+      const1 = (ConstantTreeNode)grammar.Symbols.OfType<Constant>().First().CreateTreeNode();
+      const1.Value = 1;
+      varNode = (VariableTreeNode)grammar.Symbols.OfType<Variable>().First().CreateTreeNode();
+
+      addSy = grammar.AllowedSymbols.OfType<Addition>().First();
+      mulSy = grammar.AllowedSymbols.OfType<Multiplication>().First();
+      logSy = grammar.AllowedSymbols.OfType<Logarithm>().First();
+      expSy = grammar.AllowedSymbols.OfType<Exponential>().First();
+      divSy = grammar.AllowedSymbols.OfType<Division>().First();
+
+      progRootSy = grammar.AllowedSymbols.OfType<ProgramRootSymbol>().First();
+      startSy = grammar.AllowedSymbols.OfType<StartSymbol>().First();
+    }
+
+    public ISymbolicExpressionTreeNode Exec(byte[] code, double[] consts, int nParams, double[] scalingFactor, double[] scalingOffset) {
+      int topOfStack = -1;
+      int pc = 0;
+      int nextParamIdx = -1;
+      OpCodes op;
+      short arg;
+      while (true) {
+        ReadNext(code, ref pc, out op, out arg);
+        switch (op) {
+          case OpCodes.Nop: break;
+          case OpCodes.LoadConst0: {
+              ++topOfStack;
+              stack[topOfStack] = (ISymbolicExpressionTreeNode)const0.Clone();
+              break;
+            }
+          case OpCodes.LoadConst1: {
+              ++topOfStack;
+              stack[topOfStack] = (ISymbolicExpressionTreeNode)const1.Clone();
+              break;
+            }
+          case OpCodes.LoadParamN: {
+              ++topOfStack;
+              var p = (ConstantTreeNode)const1.Clone(); // value will be tuned later (evaluator and tree generator both use 1 as initial values)
+              p.Value = consts[++nextParamIdx];
+              stack[topOfStack] = p;
+              break;
+            }
+          case OpCodes.LoadVar:
+            ++topOfStack;
+            if (scalingOffset != null) {
+              var sumNode = addSy.CreateTreeNode();
+              var varNode = (VariableTreeNode)this.varNode.Clone();
+              var constNode = (ConstantTreeNode)const0.Clone();
+              varNode.Weight = scalingFactor[arg];
+              varNode.VariableName = variableNames[arg];
+              constNode.Value = scalingOffset[arg];
+              sumNode.AddSubtree(varNode);
+              sumNode.AddSubtree(constNode);
+              stack[topOfStack] = sumNode;
+            } else {
+              var varNode = (VariableTreeNode)this.varNode.Clone();
+              varNode.Weight = 1.0;
+              varNode.VariableName = variableNames[arg];
+              stack[topOfStack] = varNode;
+            }
+            break;
+          case OpCodes.Add: {
+              var t1 = stack[topOfStack - 1];
+              var t2 = stack[topOfStack];
+              topOfStack--;
+              if (t1.Symbol is Addition) {
+                t1.AddSubtree(t2);
+              } else {
+                var addNode = addSy.CreateTreeNode();
+                addNode.AddSubtree(t1);
+                addNode.AddSubtree(t2);
+                stack[topOfStack] = addNode;
+              }
+              break;
+            }
+          case OpCodes.Mul: {
+              var t1 = stack[topOfStack - 1];
+              var t2 = stack[topOfStack];
+              topOfStack--;
+              if (t1.Symbol is Multiplication) {
+                t1.AddSubtree(t2);
+              } else {
+                var mulNode = mulSy.CreateTreeNode();
+                mulNode.AddSubtree(t1);
+                mulNode.AddSubtree(t2);
+                stack[topOfStack] = mulNode;
+              }
+              break;
+            }
+          case OpCodes.Log: {
+              var v1 = stack[topOfStack];
+              var logNode = logSy.CreateTreeNode();
+              logNode.AddSubtree(v1);
+              stack[topOfStack] = logNode;
+              break;
+            }
+          case OpCodes.Exp: {
+              var v1 = stack[topOfStack];
+              var expNode = expSy.CreateTreeNode();
+              expNode.AddSubtree(v1);
+              stack[topOfStack] = expNode;
+              break;
+            }
+          case OpCodes.Inv: {
+              var v1 = stack[topOfStack];
+              var divNode = divSy.CreateTreeNode();
+              divNode.AddSubtree(v1);
+              stack[topOfStack] = divNode;
+              break;
+            }
+          case OpCodes.Exit:
+            Contract.Assert(topOfStack == 0);
+            var rootNode = progRootSy.CreateTreeNode();
+            var startNode = startSy.CreateTreeNode();
+            startNode.AddSubtree(stack[topOfStack]);
+            rootNode.AddSubtree(startNode);
+            return rootNode;
+        }
+      }
+    }
+
+    private void ReadNext(byte[] code, ref int pc, out OpCodes op, out short s) {
+      op = (OpCodes)Enum.ToObject(typeof(OpCodes), code[pc++]);
+      s = 0;
+      if (op == OpCodes.LoadVar) {
+        s = (short)(((short)code[pc] << 8) | (short)code[pc + 1]);
+        pc += 2;
+      }
+    }
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Tree.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Tree.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/Tree.cs	(revision 14029)
@@ -0,0 +1,35 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * 
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression.Policies;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression {
+  // represents tree nodes for the search tree in MCTS
+  internal class Tree {
+    public int state;
+    public bool Done {
+      get { return actionStatistics.Done; }
+      set { actionStatistics.Done = value; }
+    }
+    public IActionStatistics actionStatistics;
+    public Tree[] children;
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaClassificationSolution.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaClassificationSolution.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaClassificationSolution.cs	(revision 14029)
@@ -40,5 +40,5 @@
       : base(original, cloner) {
     }
-    public NcaClassificationSolution(IClassificationProblemData problemData, INcaModel ncaModel)
+    public NcaClassificationSolution(INcaModel ncaModel, IClassificationProblemData problemData)
       : base(ncaModel, problemData) {
     }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs	(revision 14029)
@@ -30,5 +30,8 @@
   [Item("NCA Model", "")]
   [StorableClass]
-  public class NcaModel : NamedItem, INcaModel {
+  public class NcaModel : ClassificationModel, INcaModel {
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return allowedInputVariables; }
+    }
 
     [Storable]
@@ -39,6 +42,4 @@
     [Storable]
     private string[] allowedInputVariables;
-    [Storable]
-    private string targetVariable;
     [Storable]
     private INearestNeighbourModel nnModel;
@@ -52,14 +53,13 @@
       this.transformationMatrix = (double[,])original.transformationMatrix.Clone();
       this.allowedInputVariables = (string[])original.allowedInputVariables.Clone();
-      this.targetVariable = original.targetVariable;
       this.nnModel = cloner.Clone(original.nnModel);
       this.classValues = (double[])original.classValues.Clone();
     }
-    public NcaModel(int k, double[,] transformationMatrix, IDataset dataset, IEnumerable<int> rows, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues) {
+    public NcaModel(int k, double[,] transformationMatrix, IDataset dataset, IEnumerable<int> rows, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues)
+      : base(targetVariable) {
       Name = ItemName;
       Description = ItemDescription;
       this.transformationMatrix = (double[,])transformationMatrix.Clone();
       this.allowedInputVariables = allowedInputVariables.ToArray();
-      this.targetVariable = targetVariable;
       this.classValues = (double[])classValues.Clone();
 
@@ -72,15 +72,15 @@
     }
 
-    public IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
       var ds = ReduceDataset(dataset, rows);
       return nnModel.GetEstimatedClassValues(ds, Enumerable.Range(0, ds.Rows));
     }
 
-    public INcaClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
-      return new NcaClassificationSolution(new ClassificationProblemData(problemData), this);
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new NcaClassificationSolution(this, new ClassificationProblemData(problemData));
     }
 
-    IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {
-      return CreateClassificationSolution(problemData);
+    INcaClassificationSolution INcaModel.CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new NcaClassificationSolution(this, new ClassificationProblemData(problemData));
     }
 
@@ -88,5 +88,5 @@
       var data = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
 
-      var targets = dataset.GetDoubleValues(targetVariable, rows).ToArray();
+      var targets = dataset.GetDoubleValues(TargetVariable, rows).ToArray();
       var result = new double[data.GetLength(0), transformationMatrix.GetLength(1) + 1];
       for (int i = 0; i < data.GetLength(0); i++)
@@ -104,5 +104,5 @@
           .Range(0, transformationMatrix.GetLength(1))
           .Select(x => "X" + x.ToString())
-          .Concat(targetVariable.ToEnumerable()),
+          .Concat(TargetVariable.ToEnumerable()),
         Reduce(dataset, rows));
     }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs	(revision 14029)
@@ -34,5 +34,5 @@
   /// Nearest neighbour classification data analysis algorithm.
   /// </summary>
-  [Item("Nearest Neighbour Classification", "Nearest neighbour classification data analysis algorithm (wrapper for ALGLIB).")]
+  [Item("Nearest Neighbour Classification (kNN)", "Nearest neighbour classification data analysis algorithm (wrapper for ALGLIB).")]
   [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 150)]
   [StorableClass]
@@ -81,5 +81,5 @@
     public static IClassificationSolution CreateNearestNeighbourClassificationSolution(IClassificationProblemData problemData, int k) {
       var problemDataClone = (IClassificationProblemData)problemData.Clone();
-      return new NearestNeighbourClassificationSolution(problemDataClone, Train(problemDataClone, k));
+      return new NearestNeighbourClassificationSolution(Train(problemDataClone, k), problemDataClone);
     }
 
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassificationSolution.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassificationSolution.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassificationSolution.cs	(revision 14029)
@@ -43,5 +43,5 @@
       : base(original, cloner) {
     }
-    public NearestNeighbourClassificationSolution(IClassificationProblemData problemData, INearestNeighbourModel nnModel)
+    public NearestNeighbourClassificationSolution(INearestNeighbourModel nnModel, IClassificationProblemData problemData)
       : base(nnModel, problemData) {
     }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs	(revision 14029)
@@ -34,5 +34,5 @@
   [StorableClass]
   [Item("NearestNeighbourModel", "Represents a nearest neighbour model for regression and classification.")]
-  public sealed class NearestNeighbourModel : NamedItem, INearestNeighbourModel {
+  public sealed class NearestNeighbourModel : ClassificationModel, INearestNeighbourModel {
 
     private alglib.nearestneighbor.kdtree kdTree;
@@ -48,6 +48,8 @@
     }
 
-    [Storable]
-    private string targetVariable;
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return allowedInputVariables; }
+    }
+
     [Storable]
     private string[] allowedInputVariables;
@@ -91,14 +93,13 @@
 
       k = original.k;
-      targetVariable = original.targetVariable;
       allowedInputVariables = (string[])original.allowedInputVariables.Clone();
       if (original.classValues != null)
         this.classValues = (double[])original.classValues.Clone();
     }
-    public NearestNeighbourModel(IDataset dataset, IEnumerable<int> rows, int k, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues = null) {
+    public NearestNeighbourModel(IDataset dataset, IEnumerable<int> rows, int k, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues = null)
+      : base(targetVariable) {
       Name = ItemName;
       Description = ItemDescription;
       this.k = k;
-      this.targetVariable = targetVariable;
       this.allowedInputVariables = allowedInputVariables.ToArray();
 
@@ -163,5 +164,5 @@
     }
 
-    public IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
       if (classValues == null) throw new InvalidOperationException("No class values are defined.");
       double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
@@ -201,15 +202,10 @@
     }
 
-    public INearestNeighbourRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
-      return new NearestNeighbourRegressionSolution(new RegressionProblemData(problemData), this);
-    }
+
     IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
-      return CreateRegressionSolution(problemData);
-    }
-    public INearestNeighbourClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
-      return new NearestNeighbourClassificationSolution(new ClassificationProblemData(problemData), this);
-    }
-    IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {
-      return CreateClassificationSolution(problemData);
+      return new NearestNeighbourRegressionSolution(this, new RegressionProblemData(problemData));
+    }
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new NearestNeighbourClassificationSolution(this, new ClassificationProblemData(problemData));
     }
 
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs	(revision 14029)
@@ -33,5 +33,5 @@
   /// Nearest neighbour regression data analysis algorithm.
   /// </summary>
-  [Item("Nearest Neighbour Regression", "Nearest neighbour regression data analysis algorithm (wrapper for ALGLIB).")]
+  [Item("Nearest Neighbour Regression (kNN)", "Nearest neighbour regression data analysis algorithm (wrapper for ALGLIB).")]
   [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 150)]
   [StorableClass]
@@ -80,5 +80,5 @@
     public static IRegressionSolution CreateNearestNeighbourRegressionSolution(IRegressionProblemData problemData, int k) {
       var clonedProblemData = (IRegressionProblemData)problemData.Clone();
-      return new NearestNeighbourRegressionSolution(clonedProblemData, Train(problemData, k));
+      return new NearestNeighbourRegressionSolution(Train(problemData, k), clonedProblemData);
     }
 
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegressionSolution.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegressionSolution.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegressionSolution.cs	(revision 14029)
@@ -43,5 +43,5 @@
       : base(original, cloner) {
     }
-    public NearestNeighbourRegressionSolution(IRegressionProblemData problemData, INearestNeighbourModel nnModel)
+    public NearestNeighbourRegressionSolution(INearestNeighbourModel nnModel, IRegressionProblemData problemData)
       : base(nnModel, problemData) {
     }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs	(revision 14029)
@@ -35,5 +35,5 @@
   /// Neural network classification data analysis algorithm.
   /// </summary>
-  [Item("Neural Network Classification", "Neural network classification data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/neuralnetworks.php")]
+  [Item("Neural Network Classification (NN)", "Neural network classification data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/neuralnetworks.php")]
   [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 130)]
   [StorableClass]
@@ -220,5 +220,5 @@
 
       var problemDataClone = (IClassificationProblemData)problemData.Clone();
-      return new NeuralNetworkClassificationSolution(problemDataClone, new NeuralNetworkModel(multiLayerPerceptron, targetVariable, allowedInputVariables, problemDataClone.ClassValues.ToArray()));
+      return new NeuralNetworkClassificationSolution(new NeuralNetworkModel(multiLayerPerceptron, targetVariable, allowedInputVariables, problemDataClone.ClassValues.ToArray()), problemDataClone);
     }
     #endregion
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassificationSolution.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassificationSolution.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassificationSolution.cs	(revision 14029)
@@ -43,5 +43,5 @@
       : base(original, cloner) {
     }
-    public NeuralNetworkClassificationSolution(IClassificationProblemData problemData, INeuralNetworkModel nnModel)
+    public NeuralNetworkClassificationSolution(INeuralNetworkModel nnModel, IClassificationProblemData problemData)
       : base(nnModel, problemData) {
     }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassification.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassification.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassification.cs	(revision 14029)
@@ -35,5 +35,5 @@
   /// Neural network ensemble classification data analysis algorithm.
   /// </summary>
-  [Item("Neural Network Ensemble Classification", "Neural network ensemble classification data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/mlpensembles.php")]
+  [Item("Neural Network Ensemble Classification (NN)", "Neural network ensemble classification data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/mlpensembles.php")]
   [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 140)]
   [StorableClass]
@@ -204,5 +204,5 @@
       relClassError = alglib.mlperelclserror(mlpEnsemble, inputMatrix, nRows);
       var problemDataClone = (IClassificationProblemData)problemData.Clone();
-      return new NeuralNetworkEnsembleClassificationSolution(problemDataClone, new NeuralNetworkEnsembleModel(mlpEnsemble, targetVariable, allowedInputVariables, problemDataClone.ClassValues.ToArray()));
+      return new NeuralNetworkEnsembleClassificationSolution(new NeuralNetworkEnsembleModel(mlpEnsemble, targetVariable, allowedInputVariables, problemDataClone.ClassValues.ToArray()), problemDataClone);
     }
     #endregion
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassificationSolution.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassificationSolution.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassificationSolution.cs	(revision 14029)
@@ -43,5 +43,5 @@
       : base(original, cloner) {
     }
-    public NeuralNetworkEnsembleClassificationSolution(IClassificationProblemData problemData, INeuralNetworkEnsembleModel nnModel)
+    public NeuralNetworkEnsembleClassificationSolution(INeuralNetworkEnsembleModel nnModel, IClassificationProblemData problemData)
       : base(nnModel, problemData) {
     }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleModel.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleModel.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleModel.cs	(revision 14029)
@@ -34,5 +34,5 @@
   [StorableClass]
   [Item("NeuralNetworkEnsembleModel", "Represents a neural network ensemble for regression and classification.")]
-  public sealed class NeuralNetworkEnsembleModel : NamedItem, INeuralNetworkEnsembleModel {
+  public sealed class NeuralNetworkEnsembleModel : ClassificationModel, INeuralNetworkEnsembleModel {
 
     private alglib.mlpensemble mlpEnsemble;
@@ -46,4 +46,8 @@
         }
       }
+    }
+
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return allowedInputVariables; }
     }
 
@@ -72,5 +76,5 @@
     }
     public NeuralNetworkEnsembleModel(alglib.mlpensemble mlpEnsemble, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues = null)
-      : base() {
+      : base(targetVariable) {
       this.name = ItemName;
       this.description = ItemDescription;
@@ -103,5 +107,5 @@
     }
 
-    public IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
       double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
 
@@ -129,15 +133,9 @@
     }
 
-    public INeuralNetworkEnsembleRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
-      return new NeuralNetworkEnsembleRegressionSolution(new RegressionEnsembleProblemData(problemData), this);
-    }
-    IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
-      return CreateRegressionSolution(problemData);
-    }
-    public INeuralNetworkEnsembleClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
-      return new NeuralNetworkEnsembleClassificationSolution(new ClassificationEnsembleProblemData(problemData), this);
-    }
-    IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {
-      return CreateClassificationSolution(problemData);
+    public IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new NeuralNetworkEnsembleRegressionSolution(this, new RegressionEnsembleProblemData(problemData));
+    }
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new NeuralNetworkEnsembleClassificationSolution(this, new ClassificationEnsembleProblemData(problemData));
     }
 
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegression.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegression.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegression.cs	(revision 14029)
@@ -35,5 +35,5 @@
   /// Neural network ensemble regression data analysis algorithm.
   /// </summary>
-  [Item("Neural Network Ensemble Regression", "Neural network ensemble regression data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/mlpensembles.php")]
+  [Item("Neural Network Ensemble Regression (NN)", "Neural network ensemble regression data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/mlpensembles.php")]
   [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 140)]
   [StorableClass]
@@ -190,5 +190,5 @@
       avgRelError = alglib.mlpeavgrelerror(mlpEnsemble, inputMatrix, nRows);
 
-      return new NeuralNetworkEnsembleRegressionSolution((IRegressionProblemData)problemData.Clone(), new NeuralNetworkEnsembleModel(mlpEnsemble, targetVariable, allowedInputVariables));
+      return new NeuralNetworkEnsembleRegressionSolution(new NeuralNetworkEnsembleModel(mlpEnsemble, targetVariable, allowedInputVariables), (IRegressionProblemData)problemData.Clone());
     }
     #endregion
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegressionSolution.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegressionSolution.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegressionSolution.cs	(revision 14029)
@@ -43,5 +43,5 @@
       : base(original, cloner) {
     }
-    public NeuralNetworkEnsembleRegressionSolution(IRegressionProblemData problemData, INeuralNetworkEnsembleModel nnModel)
+    public NeuralNetworkEnsembleRegressionSolution(INeuralNetworkEnsembleModel nnModel, IRegressionProblemData problemData)
       : base(nnModel, problemData) {
       RecalculateResults();
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkModel.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkModel.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkModel.cs	(revision 14029)
@@ -34,5 +34,5 @@
   [StorableClass]
   [Item("NeuralNetworkModel", "Represents a neural network for regression and classification.")]
-  public sealed class NeuralNetworkModel : NamedItem, INeuralNetworkModel {
+  public sealed class NeuralNetworkModel : ClassificationModel, INeuralNetworkModel {
 
     private alglib.multilayerperceptron multiLayerPerceptron;
@@ -48,6 +48,8 @@
     }
 
-    [Storable]
-    private string targetVariable;
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return allowedInputVariables; }
+    }
+
     [Storable]
     private string[] allowedInputVariables;
@@ -74,5 +76,4 @@
       multiLayerPerceptron.innerobj.x = (double[])original.multiLayerPerceptron.innerobj.x.Clone();
       multiLayerPerceptron.innerobj.y = (double[])original.multiLayerPerceptron.innerobj.y.Clone();
-      targetVariable = original.targetVariable;
       allowedInputVariables = (string[])original.allowedInputVariables.Clone();
       if (original.classValues != null)
@@ -80,9 +81,8 @@
     }
     public NeuralNetworkModel(alglib.multilayerperceptron multiLayerPerceptron, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues = null)
-      : base() {
+      : base(targetVariable) {
       this.name = ItemName;
       this.description = ItemDescription;
       this.multiLayerPerceptron = multiLayerPerceptron;
-      this.targetVariable = targetVariable;
       this.allowedInputVariables = allowedInputVariables.ToArray();
       if (classValues != null)
@@ -111,5 +111,5 @@
     }
 
-    public IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
       double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
 
@@ -137,15 +137,9 @@
     }
 
-    public INeuralNetworkRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
-      return new NeuralNetworkRegressionSolution(new RegressionProblemData(problemData), this);
-    }
-    IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
-      return CreateRegressionSolution(problemData);
-    }
-    public INeuralNetworkClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
-      return new NeuralNetworkClassificationSolution(new ClassificationProblemData(problemData), this);
-    }
-    IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {
-      return CreateClassificationSolution(problemData);
+    public IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new NeuralNetworkRegressionSolution(this, new RegressionProblemData(problemData));
+    }
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new NeuralNetworkClassificationSolution(this, new ClassificationProblemData(problemData));
     }
 
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs	(revision 14029)
@@ -35,5 +35,5 @@
   /// Neural network regression data analysis algorithm.
   /// </summary>
-  [Item("Neural Network Regression", "Neural network regression data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/neuralnetworks.php")]
+  [Item("Neural Network Regression (NN)", "Neural network regression data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/neuralnetworks.php")]
   [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 130)]
   [StorableClass]
@@ -207,5 +207,5 @@
       avgRelError = alglib.mlpavgrelerror(multiLayerPerceptron, inputMatrix, nRows);
 
-      return new NeuralNetworkRegressionSolution((IRegressionProblemData)problemData.Clone(), new NeuralNetworkModel(multiLayerPerceptron, targetVariable, allowedInputVariables));
+      return new NeuralNetworkRegressionSolution(new NeuralNetworkModel(multiLayerPerceptron, targetVariable, allowedInputVariables), (IRegressionProblemData)problemData.Clone());
     }
     #endregion
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegressionSolution.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegressionSolution.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegressionSolution.cs	(revision 14029)
@@ -43,5 +43,5 @@
       : base(original, cloner) {
     }
-    public NeuralNetworkRegressionSolution(IRegressionProblemData problemData, INeuralNetworkModel nnModel)
+    public NeuralNetworkRegressionSolution(INeuralNetworkModel nnModel, IRegressionProblemData problemData)
       : base(nnModel, problemData) {
     }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NonlinearRegression/NonlinearRegression.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NonlinearRegression/NonlinearRegression.cs	(revision 14029)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/NonlinearRegression/NonlinearRegression.cs	(revision 14029)
@@ -0,0 +1,114 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Nonlinear regression data analysis algorithm.
+  /// </summary>
+  [Item("Nonlinear Regression (NLR)", "Nonlinear regression (curve fitting) data analysis algorithm (wrapper for ALGLIB).")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 120)]
+  [StorableClass]
+  public sealed class NonlinearRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+    private const string LinearRegressionModelResultName = "Regression solution";
+    private const string ModelStructureParameterName = "Model structure";
+    private const string IterationsParameterName = "Iterations";
+
+    public IFixedValueParameter<StringValue> ModelStructureParameter {
+      get { return (IFixedValueParameter<StringValue>)Parameters[ModelStructureParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> IterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[IterationsParameterName]; }
+    }
+
+    public string ModelStructure {
+      get { return ModelStructureParameter.Value.Value; }
+      set { ModelStructureParameter.Value.Value = value; }
+    }
+
+    public int Iterations {
+      get { return IterationsParameter.Value.Value; }
+      set { IterationsParameter.Value.Value = value; }
+    }
+
+
+    [StorableConstructor]
+    private NonlinearRegression(bool deserializing) : base(deserializing) { }
+    private NonlinearRegression(NonlinearRegression original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NonlinearRegression()
+      : base() {
+      Problem = new RegressionProblem();
+      Parameters.Add(new FixedValueParameter<StringValue>(ModelStructureParameterName, "The function for which the parameters must be fit (only numeric constants are tuned).", new StringValue("1.0 * x*x + 0.0")));
+      Parameters.Add(new FixedValueParameter<IntValue>(IterationsParameterName, "The maximum number of iterations for constants optimization.", new IntValue(200)));
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NonlinearRegression(this, cloner);
+    }
+
+    #region nonlinear regression
+    protected override void Run() {
+      var solution = CreateRegressionSolution(Problem.ProblemData, ModelStructure, Iterations);
+      Results.Add(new Result(LinearRegressionModelResultName, "The nonlinear regression solution.", solution));
+      Results.Add(new Result("Root mean square error (train)", "The root of the mean of squared errors of the regression solution on the training set.", new DoubleValue(solution.TrainingRootMeanSquaredError)));
+      Results.Add(new Result("Root mean square error (test)", "The root of the mean of squared errors of the regression solution on the test set.", new DoubleValue(solution.TestRootMeanSquaredError)));
+    }
+
+    public static ISymbolicRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData, string modelStructure, int maxIterations) {
+      var parser = new InfixExpressionParser();
+      var tree = parser.Parse(modelStructure);
+      var simplifier = new SymbolicDataAnalysisExpressionTreeSimplifier();
+      
+      if (!SymbolicRegressionConstantOptimizationEvaluator.CanOptimizeConstants(tree)) throw new ArgumentException("The optimizer does not support the specified model structure.");
+
+      var interpreter = new SymbolicDataAnalysisExpressionTreeLinearInterpreter();
+      SymbolicRegressionConstantOptimizationEvaluator.OptimizeConstants(interpreter, tree, problemData, problemData.TrainingIndices, 
+        applyLinearScaling: false, maxIterations: maxIterations,
+        updateVariableWeights: false, updateConstantsInTree: true);
+
+
+      var scaledModel = new SymbolicRegressionModel(problemData.TargetVariable, tree, (ISymbolicDataAnalysisExpressionTreeInterpreter)interpreter.Clone());
+      scaledModel.Scale(problemData);
+      SymbolicRegressionSolution solution = new SymbolicRegressionSolution(scaledModel, (IRegressionProblemData)problemData.Clone());
+      solution.Model.Name = "Regression Model";
+      solution.Name = "Regression Solution";
+      return solution;
+    }
+    #endregion
+  }
+}
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame	(revision 14029)
@@ -26,7 +26,8 @@
   /// Plugin class for HeuristicLab.Algorithms.DataAnalysis plugin.
   /// </summary>
-  [Plugin("HeuristicLab.Algorithms.DataAnalysis", "Provides wrappers for data analysis algorithms implemented in external libraries (linear regression, linear discriminant analysis, k-means clustering, support vector classification and regression)", "3.4.8.$WCREV$")]
+  [Plugin("HeuristicLab.Algorithms.DataAnalysis", "Provides wrappers for data analysis algorithms implemented in external libraries (linear regression, linear discriminant analysis, k-means clustering, support vector classification and regression)", "3.4.9.$WCREV$")]
   [PluginFile("HeuristicLab.Algorithms.DataAnalysis-3.4.dll", PluginFileType.Assembly)]
   [PluginDependency("HeuristicLab.ALGLIB", "3.7.0")]
+  [PluginDependency("HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm", "3.3")] // for GBM
   [PluginDependency("HeuristicLab.Algorithms.GradientDescent", "3.3")]
   [PluginDependency("HeuristicLab.Analysis", "3.3")]
@@ -47,6 +48,8 @@
   [PluginDependency("HeuristicLab.Problems.DataAnalysis.Symbolic.Regression", "3.4")]
   [PluginDependency("HeuristicLab.Problems.DataAnalysis.Symbolic.TimeSeriesPrognosis", "3.4")]
+  [PluginDependency("HeuristicLab.Problems.Instances", "3.3")]
   [PluginDependency("HeuristicLab.LibSVM", "3.12")]
   [PluginDependency("HeuristicLab.Random", "3.3")]
+  [PluginDependency("HeuristicLab.Selection", "3.3")]
   public class HeuristicLabAlgorithmsDataAnalysisPlugin : PluginBase {
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Properties/AssemblyInfo.cs.frame
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Properties/AssemblyInfo.cs.frame	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/Properties/AssemblyInfo.cs.frame	(revision 14029)
@@ -53,3 +53,3 @@
 // by using the '*' as shown below:
 [assembly: AssemblyVersion("3.4.0.0")]
-[assembly: AssemblyFileVersion("3.4.8.$WCREV$")]
+[assembly: AssemblyFileVersion("3.4.9.$WCREV$")]
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassification.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassification.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassification.cs	(revision 14029)
@@ -32,5 +32,5 @@
   /// Random forest classification data analysis algorithm.
   /// </summary>
-  [Item("Random Forest Classification", "Random forest classification data analysis algorithm (wrapper for ALGLIB).")]
+  [Item("Random Forest Classification (RF)", "Random forest classification data analysis algorithm (wrapper for ALGLIB).")]
   [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 120)]
   [StorableClass]
@@ -42,4 +42,5 @@
     private const string SeedParameterName = "Seed";
     private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+    private const string CreateSolutionParameterName = "CreateSolution";
 
     #region parameter properties
@@ -58,4 +59,7 @@
     public IFixedValueParameter<BoolValue> SetSeedRandomlyParameter {
       get { return (IFixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> CreateSolutionParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
     }
     #endregion
@@ -81,4 +85,8 @@
       set { SetSeedRandomlyParameter.Value.Value = value; }
     }
+    public bool CreateSolution {
+      get { return CreateSolutionParameter.Value.Value; }
+      set { CreateSolutionParameter.Value.Value = value; }
+    }
     #endregion
 
@@ -96,4 +104,7 @@
       Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
       Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+      Parameters[CreateSolutionParameterName].Hidden = true;
+
       Problem = new ClassificationProblem();
     }
@@ -101,4 +112,6 @@
     [StorableHook(HookType.AfterDeserialization)]
     private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
       if (!Parameters.ContainsKey(MParameterName))
         Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName, "The ratio of features that will be used in the construction of individual trees (0<m<=1)", new DoubleValue(0.5)));
@@ -107,4 +120,9 @@
       if (!Parameters.ContainsKey((SetSeedRandomlyParameterName)))
         Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+      if (!Parameters.ContainsKey(CreateSolutionParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+        Parameters[CreateSolutionParameterName].Hidden = true;
+      }
+      #endregion
     }
 
@@ -118,16 +136,26 @@
       if (SetSeedRandomly) Seed = new System.Random().Next();
 
-      var solution = CreateRandomForestClassificationSolution(Problem.ProblemData, NumberOfTrees, R, M, Seed, out rmsError, out relClassificationError, out outOfBagRmsError, out outOfBagRelClassificationError);
-      Results.Add(new Result(RandomForestClassificationModelResultName, "The random forest classification solution.", solution));
+      var model = CreateRandomForestClassificationModel(Problem.ProblemData, NumberOfTrees, R, M, Seed, out rmsError, out relClassificationError, out outOfBagRmsError, out outOfBagRelClassificationError);
       Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the random forest regression solution on the training set.", new DoubleValue(rmsError)));
       Results.Add(new Result("Relative classification error", "Relative classification error of the random forest regression solution on the training set.", new PercentValue(relClassificationError)));
       Results.Add(new Result("Root mean square error (out-of-bag)", "The out-of-bag root of the mean of squared errors of the random forest regression solution.", new DoubleValue(outOfBagRmsError)));
       Results.Add(new Result("Relative classification error (out-of-bag)", "The out-of-bag relative classification error  of the random forest regression solution.", new PercentValue(outOfBagRelClassificationError)));
+
+      if (CreateSolution) {
+        var solution = new RandomForestClassificationSolution(model, (IClassificationProblemData)Problem.ProblemData.Clone());
+        Results.Add(new Result(RandomForestClassificationModelResultName, "The random forest classification solution.", solution));
+      }
     }
 
-    public static IClassificationSolution CreateRandomForestClassificationSolution(IClassificationProblemData problemData, int nTrees, double r, double m, int seed,
+    // keep for compatibility with old API
+    public static RandomForestClassificationSolution CreateRandomForestClassificationSolution(IClassificationProblemData problemData, int nTrees, double r, double m, int seed,
       out double rmsError, out double relClassificationError, out double outOfBagRmsError, out double outOfBagRelClassificationError) {
-      var model = RandomForestModel.CreateClassificationModel(problemData, nTrees, r, m, seed, out rmsError, out relClassificationError, out outOfBagRmsError, out outOfBagRelClassificationError);
-      return new RandomForestClassificationSolution((IClassificationProblemData)problemData.Clone(), model);
+      var model = CreateRandomForestClassificationModel(problemData, nTrees, r, m, seed, out rmsError, out relClassificationError, out outOfBagRmsError, out outOfBagRelClassificationError);
+      return new RandomForestClassificationSolution(model, (IClassificationProblemData)problemData.Clone());
+    }
+
+    public static RandomForestModel CreateRandomForestClassificationModel(IClassificationProblemData problemData, int nTrees, double r, double m, int seed,
+      out double rmsError, out double relClassificationError, out double outOfBagRmsError, out double outOfBagRelClassificationError) {
+      return RandomForestModel.CreateClassificationModel(problemData, nTrees, r, m, seed, out rmsError, out relClassificationError, out outOfBagRmsError, out outOfBagRelClassificationError);
     }
     #endregion
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassificationSolution.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassificationSolution.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassificationSolution.cs	(revision 14029)
@@ -43,5 +43,5 @@
       : base(original, cloner) {
     }
-    public RandomForestClassificationSolution(IClassificationProblemData problemData, IRandomForestModel randomForestModel)
+    public RandomForestClassificationSolution(IRandomForestModel randomForestModel, IClassificationProblemData problemData)
       : base(randomForestModel, problemData) {
     }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs	(revision 14029)
@@ -34,5 +34,5 @@
   [StorableClass]
   [Item("RandomForestModel", "Represents a random forest for regression and classification.")]
-  public sealed class RandomForestModel : NamedItem, IRandomForestModel {
+  public sealed class RandomForestModel : ClassificationModel, IRandomForestModel {
     // not persisted
     private alglib.decisionforest randomForest;
@@ -44,4 +44,9 @@
       }
     }
+
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return originalTrainingData.AllowedInputVariables; }
+    }
+
 
     // instead of storing the data of the model itself
@@ -91,8 +96,8 @@
 
     // random forest models can only be created through the static factory methods CreateRegressionModel and CreateClassificationModel
-    private RandomForestModel(alglib.decisionforest randomForest,
+    private RandomForestModel(string targetVariable, alglib.decisionforest randomForest,
       int seed, IDataAnalysisProblemData originalTrainingData,
       int nTrees, double r, double m, double[] classValues = null)
-      : base() {
+      : base(targetVariable) {
       this.name = ItemName;
       this.description = ItemDescription;
@@ -147,5 +152,5 @@
     }
 
-    public IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
       double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, AllowedInputVariables, rows);
       AssertInputMatrix(inputData);
@@ -174,15 +179,10 @@
     }
 
-    public IRandomForestRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
-      return new RandomForestRegressionSolution(new RegressionProblemData(problemData), this);
-    }
-    IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
-      return CreateRegressionSolution(problemData);
-    }
-    public IRandomForestClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
-      return new RandomForestClassificationSolution(new ClassificationProblemData(problemData), this);
-    }
-    IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {
-      return CreateClassificationSolution(problemData);
+
+    public IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RandomForestRegressionSolution(this, new RegressionProblemData(problemData));
+    }
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new RandomForestClassificationSolution(this, new ClassificationProblemData(problemData));
     }
 
@@ -205,5 +205,5 @@
       outOfBagRmsError = rep.oobrmserror;
 
-      return new RandomForestModel(dForest, seed, problemData, nTrees, r, m);
+      return new RandomForestModel(problemData.TargetVariable, dForest, seed, problemData, nTrees, r, m);
     }
 
@@ -242,5 +242,5 @@
       outOfBagRelClassificationError = rep.oobrelclserror;
 
-      return new RandomForestModel(dForest, seed, problemData, nTrees, r, m, classValues);
+      return new RandomForestModel(problemData.TargetVariable, dForest, seed, problemData, nTrees, r, m, classValues);
     }
 
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegression.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegression.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegression.cs	(revision 14029)
@@ -32,5 +32,5 @@
   /// Random forest regression data analysis algorithm.
   /// </summary>
-  [Item("Random Forest Regression", "Random forest regression data analysis algorithm (wrapper for ALGLIB).")]
+  [Item("Random Forest Regression (RF)", "Random forest regression data analysis algorithm (wrapper for ALGLIB).")]
   [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 120)]
   [StorableClass]
@@ -42,4 +42,5 @@
     private const string SeedParameterName = "Seed";
     private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+    private const string CreateSolutionParameterName = "CreateSolution";
 
     #region parameter properties
@@ -58,4 +59,7 @@
     public IFixedValueParameter<BoolValue> SetSeedRandomlyParameter {
       get { return (IFixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> CreateSolutionParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
     }
     #endregion
@@ -81,4 +85,8 @@
       set { SetSeedRandomlyParameter.Value.Value = value; }
     }
+    public bool CreateSolution {
+      get { return CreateSolutionParameter.Value.Value; }
+      set { CreateSolutionParameter.Value.Value = value; }
+    }
     #endregion
     [StorableConstructor]
@@ -95,4 +103,7 @@
       Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
       Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+      Parameters[CreateSolutionParameterName].Hidden = true;
+
       Problem = new RegressionProblem();
     }
@@ -100,4 +111,6 @@
     [StorableHook(HookType.AfterDeserialization)]
     private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
       if (!Parameters.ContainsKey(MParameterName))
         Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName, "The ratio of features that will be used in the construction of individual trees (0<m<=1)", new DoubleValue(0.5)));
@@ -106,4 +119,9 @@
       if (!Parameters.ContainsKey((SetSeedRandomlyParameterName)))
         Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+      if (!Parameters.ContainsKey(CreateSolutionParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+        Parameters[CreateSolutionParameterName].Hidden = true;
+      }
+      #endregion
     }
 
@@ -116,18 +134,32 @@
       double rmsError, avgRelError, outOfBagRmsError, outOfBagAvgRelError;
       if (SetSeedRandomly) Seed = new System.Random().Next();
+      var model = CreateRandomForestRegressionModel(Problem.ProblemData, NumberOfTrees, R, M, Seed,
+        out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError);
 
-      var solution = CreateRandomForestRegressionSolution(Problem.ProblemData, NumberOfTrees, R, M, Seed, out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError);
-      Results.Add(new Result(RandomForestRegressionModelResultName, "The random forest regression solution.", solution));
       Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the random forest regression solution on the training set.", new DoubleValue(rmsError)));
       Results.Add(new Result("Average relative error", "The average of relative errors of the random forest regression solution on the training set.", new PercentValue(avgRelError)));
       Results.Add(new Result("Root mean square error (out-of-bag)", "The out-of-bag root of the mean of squared errors of the random forest regression solution.", new DoubleValue(outOfBagRmsError)));
       Results.Add(new Result("Average relative error (out-of-bag)", "The out-of-bag average of relative errors of the random forest regression solution.", new PercentValue(outOfBagAvgRelError)));
+
+      if (CreateSolution) {
+        var solution = new RandomForestRegressionSolution(model, (IRegressionProblemData)Problem.ProblemData.Clone());
+        Results.Add(new Result(RandomForestRegressionModelResultName, "The random forest regression solution.", solution));
+      }
     }
 
-    public static IRegressionSolution CreateRandomForestRegressionSolution(IRegressionProblemData problemData, int nTrees, double r, double m, int seed,
+    // keep for compatibility with old API
+    public static RandomForestRegressionSolution CreateRandomForestRegressionSolution(IRegressionProblemData problemData, int nTrees, double r, double m, int seed,
       out double rmsError, out double avgRelError, out double outOfBagRmsError, out double outOfBagAvgRelError) {
-      var model = RandomForestModel.CreateRegressionModel(problemData, nTrees, r, m, seed, out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError);
-      return new RandomForestRegressionSolution((IRegressionProblemData)problemData.Clone(), model);
+      var model = CreateRandomForestRegressionModel(problemData, nTrees, r, m, seed,
+        out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError);
+      return new RandomForestRegressionSolution(model, (IRegressionProblemData)problemData.Clone());
     }
+
+    public static RandomForestModel CreateRandomForestRegressionModel(IRegressionProblemData problemData, int nTrees,
+      double r, double m, int seed,
+      out double rmsError, out double avgRelError, out double outOfBagRmsError, out double outOfBagAvgRelError) {
+      return RandomForestModel.CreateRegressionModel(problemData, nTrees, r, m, seed, out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError);
+    }
+
     #endregion
   }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegressionSolution.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegressionSolution.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegressionSolution.cs	(revision 14029)
@@ -43,5 +43,5 @@
       : base(original, cloner) {
     }
-    public RandomForestRegressionSolution(IRegressionProblemData problemData, IRandomForestModel randomForestModel)
+    public RandomForestRegressionSolution(IRandomForestModel randomForestModel, IRegressionProblemData problemData)
       : base(randomForestModel, problemData) {
     }
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassification.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassification.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassification.cs	(revision 14029)
@@ -36,5 +36,5 @@
   /// Support vector machine classification data analysis algorithm.
   /// </summary>
-  [Item("Support Vector Classification", "Support vector machine classification data analysis algorithm (wrapper for libSVM).")]
+  [Item("Support Vector Classification (SVM)", "Support vector machine classification data analysis algorithm (wrapper for libSVM).")]
   [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 110)]
   [StorableClass]
@@ -212,5 +212,4 @@
       IEnumerable<int> rows = problemData.TrainingIndices;
 
-      //extract SVM parameters from scope and set them
       svm_parameter parameter = new svm_parameter {
         svm_type = svmType,
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineModel.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineModel.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineModel.cs	(revision 14029)
@@ -37,5 +37,9 @@
   [StorableClass]
   [Item("SupportVectorMachineModel", "Represents a support vector machine model.")]
-  public sealed class SupportVectorMachineModel : NamedItem, ISupportVectorMachineModel {
+  public sealed class SupportVectorMachineModel : ClassificationModel, ISupportVectorMachineModel {
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return allowedInputVariables; }
+    }
+
 
     private svm_model model;
@@ -83,6 +87,4 @@
 
     [Storable]
-    private string targetVariable;
-    [Storable]
     private string[] allowedInputVariables;
     [Storable]
@@ -96,5 +98,4 @@
       this.model = original.model;
       this.rangeTransform = original.rangeTransform;
-      this.targetVariable = original.targetVariable;
       this.allowedInputVariables = (string[])original.allowedInputVariables.Clone();
       if (original.classValues != null)
@@ -106,10 +107,9 @@
     }
     public SupportVectorMachineModel(svm_model model, RangeTransform rangeTransform, string targetVariable, IEnumerable<string> allowedInputVariables)
-      : base() {
+      : base(targetVariable) {
       this.name = ItemName;
       this.description = ItemDescription;
       this.model = model;
       this.rangeTransform = rangeTransform;
-      this.targetVariable = targetVariable;
       this.allowedInputVariables = allowedInputVariables.ToArray();
     }
@@ -123,14 +123,11 @@
       return GetEstimatedValuesHelper(dataset, rows);
     }
-    public SupportVectorRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+    public IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
       return new SupportVectorRegressionSolution(this, new RegressionProblemData(problemData));
     }
-    IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
-      return CreateRegressionSolution(problemData);
-    }
     #endregion
 
     #region IClassificationModel Members
-    public IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
       if (classValues == null) throw new NotSupportedException();
       // return the original class value instead of the predicted value of the model
@@ -152,14 +149,11 @@
     }
 
-    public SupportVectorClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
       return new SupportVectorClassificationSolution(this, new ClassificationProblemData(problemData));
-    }
-    IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {
-      return CreateClassificationSolution(problemData);
     }
     #endregion
     private IEnumerable<double> GetEstimatedValuesHelper(IDataset dataset, IEnumerable<int> rows) {
       // calculate predictions for the currently requested rows
-      svm_problem problem = SupportVectorMachineUtil.CreateSvmProblem(dataset, targetVariable, allowedInputVariables, rows);
+      svm_problem problem = SupportVectorMachineUtil.CreateSvmProblem(dataset, TargetVariable, allowedInputVariables, rows);
       svm_problem scaledProblem = rangeTransform.Scale(problem);
 
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegression.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegression.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegression.cs	(revision 14029)
@@ -36,5 +36,5 @@
   /// Support vector machine regression data analysis algorithm.
   /// </summary>
-  [Item("Support Vector Regression", "Support vector machine regression data analysis algorithm (wrapper for libSVM).")]
+  [Item("Support Vector Regression (SVM)", "Support vector machine regression data analysis algorithm (wrapper for libSVM).")]
   [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 110)]
   [StorableClass]
@@ -229,5 +229,4 @@
       IEnumerable<int> rows = problemData.TrainingIndices;
 
-      //extract SVM parameters from scope and set them
       svm_parameter parameter = new svm_parameter {
         svm_type = GetSvmType(svmType),
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/TimeSeries/AutoregressiveModeling.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/TimeSeries/AutoregressiveModeling.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/TimeSeries/AutoregressiveModeling.cs	(revision 14029)
@@ -34,5 +34,5 @@
 
 namespace HeuristicLab.Algorithms.DataAnalysis.TimeSeries {
-  [Item("Autoregressive Modeling", "Timeseries modeling algorithm that creates AR-N models.")]
+  [Item("Autoregressive Modeling (AR)", "Timeseries modeling algorithm that creates AR-N models.")]
   [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 130)]
   [StorableClass]
@@ -134,5 +134,5 @@
 
       var interpreter = new SymbolicTimeSeriesPrognosisExpressionTreeInterpreter(problemData.TargetVariable);
-      var model = new SymbolicTimeSeriesPrognosisModel(tree, interpreter);
+      var model = new SymbolicTimeSeriesPrognosisModel(problemData.TargetVariable, tree, interpreter);
       var solution = model.CreateTimeSeriesPrognosisSolution((ITimeSeriesPrognosisProblemData)problemData.Clone());
       return solution;
Index: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringModel.cs
===================================================================
--- branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringModel.cs	(revision 13401)
+++ branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringModel.cs	(revision 14029)
@@ -37,4 +37,8 @@
     public static new Image StaticItemImage {
       get { return HeuristicLab.Common.Resources.VSImageLibrary.Function; }
+    }
+
+    public IEnumerable<string> VariablesUsedForPrediction {
+      get { return allowedInputVariables; }
     }
 
