source: branches/2847_M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/M5Regression/m5.patch @ 16848

Interfaces/IM5Model.cs

@@ -24 +24 @@
 using HeuristicLab.Problems.DataAnalysis;
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
-  internal interface IM5Model : IRegressionModel {
+  public interface IM5Model : IRegressionModel {
     void Build(IReadOnlyList<int> trainingRows, IReadOnlyList<int> pruningRows, M5Parameters m5Params, CancellationToken cancellationToken);
     void Update(IReadOnlyList<int> rows, M5Parameters m5Parameters, CancellationToken cancellationToken);
   }

Interfaces/IPruning.cs

@@ -19 +19 @@
  */
 #endregion
 using System.Collections.Generic;
+using System.Threading;
 using HeuristicLab.Core;
 using HeuristicLab.Problems.DataAnalysis;
 
@@ -25 +26 @@
 namespace HeuristicLab.Algorithms.DataAnalysis {
   public interface IPruning : IParameterizedNamedItem {
     int MinLeafSize(IRegressionProblemData pd, ILeafModel leafModel);
+
+    void Prune(M5TreeModel treeModel, IReadOnlyList<int> trainingRows, IReadOnlyList<int> pruningRows, M5Parameters m5Params, CancellationToken cancellationToken);
   }
 }
+ No newline at end of file

Interfaces/ISplitter.cs

@@ +1 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public interface ISplitter : IParameterizedNamedItem {
+    /// <summary>
+    /// decides wether a node sould be split
+    /// and if so at which attribute and which value
+    /// </summary>
+    /// <param name="splitData"></param>
+    /// <param name="minLeafSize"></param>
+    /// <param name="splitAttr"></param>
+    /// <param name="splitValue"></param>
+    /// <returns></returns>
+    bool Split(IRegressionProblemData splitData, int minLeafSize, out string splitAttr, out double splitValue);
+  }
+}
+ No newline at end of file

LeafModels/ComponentReducedLinearModel.cs

@@ +1 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  public class ComponentReducedLinearModel : RegressionModel, IConfidenceRegressionModel {
+    [Storable]
+    private IConfidenceRegressionModel Model;
+    [Storable]
+    private PrincipleComponentTransformation Pca;
+
+    [StorableConstructor]
+    private ComponentReducedLinearModel(bool deserializing) : base(deserializing) { }
+    private ComponentReducedLinearModel(ComponentReducedLinearModel original, Cloner cloner) : base(original, cloner) {
+      Model = cloner.Clone(original.Model);
+      Pca = cloner.Clone(original.Pca);
+    }
+    public ComponentReducedLinearModel(string targetVariable, IConfidenceRegressionModel model, PrincipleComponentTransformation pca) : base(targetVariable) {
+      Model = model;
+      Pca = pca;
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new ComponentReducedLinearModel(this, cloner);
+    }
+
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return Model.VariablesUsedForPrediction; }
+    }
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      var data = ReduceDataset(dataset, rows.ToArray());
+      return Model.GetEstimatedValues(Pca.TransformDataset(data), Enumerable.Range(0, data.Rows));
+    }
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new ConfidenceRegressionSolution(this, problemData);
+    }
+    public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
+      var data = ReduceDataset(dataset, rows.ToArray());
+      return Model.GetEstimatedVariances(Pca.TransformDataset(data), Enumerable.Range(0, data.Rows));
+    }
+
+    private IDataset ReduceDataset(IDataset data, IReadOnlyList<int> rows) {
+      return new Dataset(data.DoubleVariables, data.DoubleVariables.Select(v => data.GetDoubleValues(v, rows).ToList()));
+    }
+  }
+}
+ No newline at end of file

LeafModels/DampenedLinearModel.cs

@@ +1 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  //mulitdimensional extension of http://www2.stat.duke.edu/~tjl13/s101/slides/unit6lec3H.pdf
+  [StorableClass]
+  public class DampenedLinearModel : RegressionModel, IConfidenceRegressionModel {
+    [Storable]
+    private IConfidenceRegressionModel Model;
+    [Storable]
+    private double Min;
+    [Storable]
+    private double Max;
+    [Storable]
+    private double Dampening;
+
+    [StorableConstructor]
+    private DampenedLinearModel(bool deserializing) : base(deserializing) { }
+    private DampenedLinearModel(DampenedLinearModel original, Cloner cloner) : base(original, cloner) {
+      Model = cloner.Clone(original.Model);
+      Min = original.Min;
+      Max = original.Max;
+      Dampening = original.Dampening;
+    }
+    public DampenedLinearModel(IConfidenceRegressionModel model, IRegressionProblemData pd, double dampening) : base(model.TargetVariable) {
+      Model = model;
+      Min = pd.TargetVariableTrainingValues.Min();
+      Max = pd.TargetVariableTrainingValues.Max();
+      Dampening = dampening;
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new DampenedLinearModel(this, cloner);
+    }
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return Model.VariablesUsedForPrediction; }
+    }
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      var slow = Sigmoid(-Dampening);
+      var shigh = Sigmoid(Dampening);
+      foreach (var x in Model.GetEstimatedValues(dataset, rows)) {
+        var y = Rescale(x, Min, Max, -Dampening, Dampening);
+        y = Sigmoid(y);
+        y = Rescale(y, slow, shigh, Min, Max);
+        yield return y;
+      }
+    }
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new ConfidenceRegressionSolution(this, problemData);
+    }
+    public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
+      return Model.GetEstimatedVariances(dataset, rows);
+    }
+
+    private static double Rescale(double x, double oMin, double oMax, double nMin, double nMax) {
+      var d = oMax - oMin;
+      var nd = nMax - nMin;
+      if (d.IsAlmost(0)) {
+        d = 1;
+        nMin += nd / 2;
+        nd = 0;
+      }
+      return ((x - oMin) / d) * nd + nMin;
+    }
+    private static double Sigmoid(double x) {
+      return 1 / (1 + Math.Exp(-x));
+    }
+  }
+}
+ No newline at end of file

LeafModels/PreconstructedLinearModel.cs

@@ +1 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  //mulitdimensional extension of http://www2.stat.duke.edu/~tjl13/s101/slides/unit6lec3H.pdf
+  [StorableClass]
+  internal sealed class PreconstructedLinearModel : RegressionModel, IConfidenceRegressionModel {
+    [Storable]
+    public Dictionary<string, double> Coefficients { get; private set; }
+    [Storable]
+    public double Intercept { get; private set; }
+    [Storable]
+    private Dictionary<string, double> Means { get; set; }
+    [Storable]
+    private Dictionary<string, double> Variances { get; set; }
+    [Storable]
+    private double ResidualVariance { get; set; }
+    [Storable]
+    private int SampleSize { get; set; }
+
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return Coefficients.Keys; }
+    }
+    #region HLConstructors
+    [StorableConstructor]
+    private PreconstructedLinearModel(bool deserializing) : base(deserializing) { }
+    private PreconstructedLinearModel(PreconstructedLinearModel original, Cloner cloner) : base(original, cloner) {
+      if (original.Coefficients != null) Coefficients = original.Coefficients.ToDictionary(x => x.Key, x => x.Value);
+      Intercept = original.Intercept;
+      if (original.Means != null) Means = original.Means.ToDictionary(x => x.Key, x => x.Value);
+      if (original.Variances != null) Variances = original.Variances.ToDictionary(x => x.Key, x => x.Value);
+      ResidualVariance = original.ResidualVariance;
+      SampleSize = original.SampleSize;
+    }
+    public PreconstructedLinearModel(Dictionary<string, double> means, Dictionary<string, double> variances, Dictionary<string, double> coefficients, double intercept, string targetvariable) : base(targetvariable) {
+      Coefficients = coefficients;
+      Intercept = intercept;
+      Variances = variances;
+      Means = means;
+      ResidualVariance = 0;
+      SampleSize = 0;
+    }
+    public PreconstructedLinearModel(double intercept, string targetvariable) : base(targetvariable) {
+      Coefficients = new Dictionary<string, double>();
+      Intercept = intercept;
+      Variances = new Dictionary<string, double>();
+      ResidualVariance = 0;
+      SampleSize = 0;
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new PreconstructedLinearModel(this, cloner);
+    }
+    #endregion
+
+    public static PreconstructedLinearModel CreateConfidenceLinearModel(IRegressionProblemData pd, out double rmse, out double cvRmse) {
+      rmse = double.NaN;
+      cvRmse = double.NaN;
+      return AlternativeCalculation(pd);
+    }
+
+    private static PreconstructedLinearModel ClassicCalculation(IRegressionProblemData pd, out double rmse, out double cvRmse) {
+      var inputMatrix = pd.Dataset.ToArray(pd.AllowedInputVariables.Concat(new[] {
+        pd.TargetVariable
+      }), pd.AllIndices);
+
+      var nFeatures = inputMatrix.GetLength(1) - 1;
+      double[] coefficients;
+
+      alglib.linearmodel lm;
+      alglib.lrreport ar;
+      int retVal;
+      alglib.lrbuild(inputMatrix, inputMatrix.GetLength(0), nFeatures, out retVal, out lm, out ar);
+      if (retVal != 1) throw new ArgumentException("Error in calculation of linear regression solution");
+      rmse = ar.rmserror;
+      cvRmse = ar.cvrmserror;
+
+      alglib.lrunpack(lm, out coefficients, out nFeatures);
+
+
+      var means = pd.AllowedInputVariables.ToDictionary(n => n, n => pd.Dataset.GetDoubleValues(n).Average());
+      var variances = pd.AllowedInputVariables.ToDictionary(n => n, n => pd.Dataset.GetDoubleValues(n).Variance());
+      var coeffs = pd.AllowedInputVariables.Zip(coefficients, (s, d) => new {s, d}).ToDictionary(x => x.s, x => x.d);
+      var res = new PreconstructedLinearModel(means, variances, coeffs, coefficients[nFeatures], pd.TargetVariable);
+
+      res.ResidualVariance = pd.TargetVariableValues.Zip(res.GetEstimatedValues(pd.Dataset, pd.TrainingIndices), (x, y) => x - y).Variance();
+      res.SampleSize = pd.TrainingIndices.Count();
+      return res;
+    }
+
+    private static PreconstructedLinearModel AlternativeCalculation(IRegressionProblemData pd) {
+      var means = pd.AllowedInputVariables.ToDictionary(n1 => n1, n1 => pd.Dataset.GetDoubleValues(n1).Average());
+      var variances = pd.AllowedInputVariables.ToDictionary(n1 => n1, n1 => pd.Dataset.GetDoubleValues(n1).Variance());
+      var cmean = pd.TargetVariableTrainingValues.Average();
+      var variables = pd.AllowedInputVariables.ToList();
+      var n = variables.Count;
+      var m = pd.TrainingIndices.Count();
+
+      //Set up X^T and y
+      var inTr = new double[n + 1, m];
+      for (var i = 0; i < n; i++) {
+        var v = variables[i];
+        var vdata = pd.Dataset.GetDoubleValues(v, pd.TrainingIndices).ToArray();
+        for (var j = 0; j < m; j++) inTr[i, j] = vdata[j];
+      }
+
+      for (var i = 0; i < m; i++) inTr[n, i] = 1;
+
+      var y = new double[m, 1];
+      var ydata = pd.TargetVariableTrainingValues.ToArray();
+      for (var i = 0; i < m; i++) y[i, 0] = ydata[i];
+
+      //Perform linear regression
+      var aTy = new double[n + 1, 1];
+      alglib.rmatrixgemm(n + 1, 1, m, 1, inTr, 0, 0, 0, y, 0, 0, 0, 0, ref aTy, 0, 0); //aTy = inTr * y;
+      var aTa = new double[n + 1, n + 1];
+      alglib.rmatrixgemm(n + 1, n + 1, m, 1, inTr, 0, 0, 0, inTr, 0, 0, 1, 0, ref aTa, 0, 0); //aTa = inTr * t(inTr) +aTa //
+      alglib.spdmatrixcholesky(ref aTa, n + 1, true);
+      int info;
+      alglib.densesolverreport report;
+      double[] coefficients;
+      var aTyVector = new double[n + 1];
+      for (var i = 0; i < n + 1; i++) aTyVector[i] = aTy[i, 0];
+      alglib.spdmatrixcholeskysolve(aTa, n + 1, true, aTyVector, out info, out report, out coefficients);
+      double rmse, cvrmse;
+      if (info != 1) return ClassicCalculation(pd, out rmse, out cvrmse);
+
+      //extract coefficients
+      var intercept = coefficients[n];
+      var coeffs = new Dictionary<string, double>();
+      for (var i = 0; i < n; i++) coeffs.Add(variables[i], coefficients[i]);
+
+      return new PreconstructedLinearModel(means, variances, coeffs, intercept, pd.TargetVariable);
+    }
+
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      return rows.Select(row => GetEstimatedValue(dataset, row));
+    }
+
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RegressionSolution(this, problemData);
+    }
+
+    public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
+      return rows.Select(i => GetEstimatedVariance(dataset, i));
+    }
+
+    #region helpers
+    private double GetEstimatedValue(IDataset dataset, int row) {
+      return Intercept + (Coefficients.Count == 0 ? 0 : Coefficients.Sum(s => s.Value * dataset.GetDoubleValue(s.Key, row)));
+    }
+    private double GetEstimatedVariance(IDataset dataset, int row) {
+      if (SampleSize == 0) return 0.0;
+      var sum = (from var in Variances let d = dataset.GetDoubleValue(var.Key, row) - Means[var.Key] select d * d / var.Value).Sum();
+      var res = ResidualVariance * (1.0 / SampleSize + sum / (SampleSize - 1));
+      if (double.IsInfinity(res) || double.IsNaN(res)) return 0.0;
+      return res;
+    }
+    #endregion
+  }
+}
+ No newline at end of file

LeafModels/ComponentReducedLinearModel.cs

@@ +1 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  public class ComponentReducedLinearModel : RegressionModel, IConfidenceRegressionModel {
+    [Storable]
+    private IConfidenceRegressionModel Model;
+    [Storable]
+    private PrincipleComponentTransformation Pca;
+
+    [StorableConstructor]
+    private ComponentReducedLinearModel(bool deserializing) : base(deserializing) { }
+    private ComponentReducedLinearModel(ComponentReducedLinearModel original, Cloner cloner) : base(original, cloner) {
+      Model = cloner.Clone(original.Model);
+      Pca = cloner.Clone(original.Pca);
+    }
+    public ComponentReducedLinearModel(string targetVariable, IConfidenceRegressionModel model, PrincipleComponentTransformation pca) : base(targetVariable) {
+      Model = model;
+      Pca = pca;
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new ComponentReducedLinearModel(this, cloner);
+    }
+
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return Model.VariablesUsedForPrediction; }
+    }
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      var data = ReduceDataset(dataset, rows.ToArray());
+      return Model.GetEstimatedValues(Pca.TransformDataset(data), Enumerable.Range(0, data.Rows));
+    }
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new ConfidenceRegressionSolution(this, problemData);
+    }
+    public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
+      var data = ReduceDataset(dataset, rows.ToArray());
+      return Model.GetEstimatedVariances(Pca.TransformDataset(data), Enumerable.Range(0, data.Rows));
+    }
+
+    private IDataset ReduceDataset(IDataset data, IReadOnlyList<int> rows) {
+      return new Dataset(data.DoubleVariables, data.DoubleVariables.Select(v => data.GetDoubleValues(v, rows).ToList()));
+    }
+  }
+}
+ No newline at end of file

LeafModels/DampenedLinearModel.cs

@@ +1 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  //mulitdimensional extension of http://www2.stat.duke.edu/~tjl13/s101/slides/unit6lec3H.pdf
+  [StorableClass]
+  public class DampenedLinearModel : RegressionModel, IConfidenceRegressionModel {
+    [Storable]
+    private IConfidenceRegressionModel Model;
+    [Storable]
+    private double Min;
+    [Storable]
+    private double Max;
+    [Storable]
+    private double Dampening;
+
+    [StorableConstructor]
+    private DampenedLinearModel(bool deserializing) : base(deserializing) { }
+    private DampenedLinearModel(DampenedLinearModel original, Cloner cloner) : base(original, cloner) {
+      Model = cloner.Clone(original.Model);
+      Min = original.Min;
+      Max = original.Max;
+      Dampening = original.Dampening;
+    }
+    public DampenedLinearModel(IConfidenceRegressionModel model, IRegressionProblemData pd, double dampening) : base(model.TargetVariable) {
+      Model = model;
+      Min = pd.TargetVariableTrainingValues.Min();
+      Max = pd.TargetVariableTrainingValues.Max();
+      Dampening = dampening;
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new DampenedLinearModel(this, cloner);
+    }
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return Model.VariablesUsedForPrediction; }
+    }
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      var slow = Sigmoid(-Dampening);
+      var shigh = Sigmoid(Dampening);
+      foreach (var x in Model.GetEstimatedValues(dataset, rows)) {
+        var y = Rescale(x, Min, Max, -Dampening, Dampening);
+        y = Sigmoid(y);
+        y = Rescale(y, slow, shigh, Min, Max);
+        yield return y;
+      }
+    }
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new ConfidenceRegressionSolution(this, problemData);
+    }
+    public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
+      return Model.GetEstimatedVariances(dataset, rows);
+    }
+
+    private static double Rescale(double x, double oMin, double oMax, double nMin, double nMax) {
+      var d = oMax - oMin;
+      var nd = nMax - nMin;
+      if (d.IsAlmost(0)) {
+        d = 1;
+        nMin += nd / 2;
+        nd = 0;
+      }
+      return ((x - oMin) / d) * nd + nMin;
+    }
+    private static double Sigmoid(double x) {
+      return 1 / (1 + Math.Exp(-x));
+    }
+  }
+}
+ No newline at end of file

LeafModels/PreconstructedLinearModel.cs

@@ +1 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  //mulitdimensional extension of http://www2.stat.duke.edu/~tjl13/s101/slides/unit6lec3H.pdf
+  [StorableClass]
+  internal sealed class PreconstructedLinearModel : RegressionModel, IConfidenceRegressionModel {
+    [Storable]
+    public Dictionary<string, double> Coefficients { get; private set; }
+    [Storable]
+    public double Intercept { get; private set; }
+    [Storable]
+    private Dictionary<string, double> Means { get; set; }
+    [Storable]
+    private Dictionary<string, double> Variances { get; set; }
+    [Storable]
+    private double ResidualVariance { get; set; }
+    [Storable]
+    private int SampleSize { get; set; }
+
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return Coefficients.Keys; }
+    }
+    #region HLConstructors
+    [StorableConstructor]
+    private PreconstructedLinearModel(bool deserializing) : base(deserializing) { }
+    private PreconstructedLinearModel(PreconstructedLinearModel original, Cloner cloner) : base(original, cloner) {
+      if (original.Coefficients != null) Coefficients = original.Coefficients.ToDictionary(x => x.Key, x => x.Value);
+      Intercept = original.Intercept;
+      if (original.Means != null) Means = original.Means.ToDictionary(x => x.Key, x => x.Value);
+      if (original.Variances != null) Variances = original.Variances.ToDictionary(x => x.Key, x => x.Value);
+      ResidualVariance = original.ResidualVariance;
+      SampleSize = original.SampleSize;
+    }
+    public PreconstructedLinearModel(Dictionary<string, double> means, Dictionary<string, double> variances, Dictionary<string, double> coefficients, double intercept, string targetvariable) : base(targetvariable) {
+      Coefficients = coefficients;
+      Intercept = intercept;
+      Variances = variances;
+      Means = means;
+      ResidualVariance = 0;
+      SampleSize = 0;
+    }
+    public PreconstructedLinearModel(double intercept, string targetvariable) : base(targetvariable) {
+      Coefficients = new Dictionary<string, double>();
+      Intercept = intercept;
+      Variances = new Dictionary<string, double>();
+      ResidualVariance = 0;
+      SampleSize = 0;
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new PreconstructedLinearModel(this, cloner);
+    }
+    #endregion
+
+    public static PreconstructedLinearModel CreateConfidenceLinearModel(IRegressionProblemData pd, out double rmse, out double cvRmse) {
+      rmse = double.NaN;
+      cvRmse = double.NaN;
+      return AlternativeCalculation(pd);
+    }
+
+    private static PreconstructedLinearModel ClassicCalculation(IRegressionProblemData pd, out double rmse, out double cvRmse) {
+      var inputMatrix = pd.Dataset.ToArray(pd.AllowedInputVariables.Concat(new[] {
+        pd.TargetVariable
+      }), pd.AllIndices);
+
+      var nFeatures = inputMatrix.GetLength(1) - 1;
+      double[] coefficients;
+
+      alglib.linearmodel lm;
+      alglib.lrreport ar;
+      int retVal;
+      alglib.lrbuild(inputMatrix, inputMatrix.GetLength(0), nFeatures, out retVal, out lm, out ar);
+      if (retVal != 1) throw new ArgumentException("Error in calculation of linear regression solution");
+      rmse = ar.rmserror;
+      cvRmse = ar.cvrmserror;
+
+      alglib.lrunpack(lm, out coefficients, out nFeatures);
+
+
+      var means = pd.AllowedInputVariables.ToDictionary(n => n, n => pd.Dataset.GetDoubleValues(n).Average());
+      var variances = pd.AllowedInputVariables.ToDictionary(n => n, n => pd.Dataset.GetDoubleValues(n).Variance());
+      var coeffs = pd.AllowedInputVariables.Zip(coefficients, (s, d) => new {s, d}).ToDictionary(x => x.s, x => x.d);
+      var res = new PreconstructedLinearModel(means, variances, coeffs, coefficients[nFeatures], pd.TargetVariable);
+
+      res.ResidualVariance = pd.TargetVariableValues.Zip(res.GetEstimatedValues(pd.Dataset, pd.TrainingIndices), (x, y) => x - y).Variance();
+      res.SampleSize = pd.TrainingIndices.Count();
+      return res;
+    }
+
+    private static PreconstructedLinearModel AlternativeCalculation(IRegressionProblemData pd) {
+      var means = pd.AllowedInputVariables.ToDictionary(n1 => n1, n1 => pd.Dataset.GetDoubleValues(n1).Average());
+      var variances = pd.AllowedInputVariables.ToDictionary(n1 => n1, n1 => pd.Dataset.GetDoubleValues(n1).Variance());
+      var cmean = pd.TargetVariableTrainingValues.Average();
+      var variables = pd.AllowedInputVariables.ToList();
+      var n = variables.Count;
+      var m = pd.TrainingIndices.Count();
+
+      //Set up X^T and y
+      var inTr = new double[n + 1, m];
+      for (var i = 0; i < n; i++) {
+        var v = variables[i];
+        var vdata = pd.Dataset.GetDoubleValues(v, pd.TrainingIndices).ToArray();
+        for (var j = 0; j < m; j++) inTr[i, j] = vdata[j];
+      }
+
+      for (var i = 0; i < m; i++) inTr[n, i] = 1;
+
+      var y = new double[m, 1];
+      var ydata = pd.TargetVariableTrainingValues.ToArray();
+      for (var i = 0; i < m; i++) y[i, 0] = ydata[i];
+
+      //Perform linear regression
+      var aTy = new double[n + 1, 1];
+      alglib.rmatrixgemm(n + 1, 1, m, 1, inTr, 0, 0, 0, y, 0, 0, 0, 0, ref aTy, 0, 0); //aTy = inTr * y;
+      var aTa = new double[n + 1, n + 1];
+      alglib.rmatrixgemm(n + 1, n + 1, m, 1, inTr, 0, 0, 0, inTr, 0, 0, 1, 0, ref aTa, 0, 0); //aTa = inTr * t(inTr) +aTa //
+      alglib.spdmatrixcholesky(ref aTa, n + 1, true);
+      int info;
+      alglib.densesolverreport report;
+      double[] coefficients;
+      var aTyVector = new double[n + 1];
+      for (var i = 0; i < n + 1; i++) aTyVector[i] = aTy[i, 0];
+      alglib.spdmatrixcholeskysolve(aTa, n + 1, true, aTyVector, out info, out report, out coefficients);
+      double rmse, cvrmse;
+      if (info != 1) return ClassicCalculation(pd, out rmse, out cvrmse);
+
+      //extract coefficients
+      var intercept = coefficients[n];
+      var coeffs = new Dictionary<string, double>();
+      for (var i = 0; i < n; i++) coeffs.Add(variables[i], coefficients[i]);
+
+      return new PreconstructedLinearModel(means, variances, coeffs, intercept, pd.TargetVariable);
+    }
+
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      return rows.Select(row => GetEstimatedValue(dataset, row));
+    }
+
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RegressionSolution(this, problemData);
+    }
+
+    public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
+      return rows.Select(i => GetEstimatedVariance(dataset, i));
+    }
+
+    #region helpers
+    private double GetEstimatedValue(IDataset dataset, int row) {
+      return Intercept + (Coefficients.Count == 0 ? 0 : Coefficients.Sum(s => s.Value * dataset.GetDoubleValue(s.Key, row)));
+    }
+    private double GetEstimatedVariance(IDataset dataset, int row) {
+      if (SampleSize == 0) return 0.0;
+      var sum = (from var in Variances let d = dataset.GetDoubleValue(var.Key, row) - Means[var.Key] select d * d / var.Value).Sum();
+      var res = ResidualVariance * (1.0 / SampleSize + sum / (SampleSize - 1));
+      if (double.IsInfinity(res) || double.IsNaN(res)) return 0.0;
+      return res;
+    }
+    #endregion
+  }
+}
+ No newline at end of file

LeafTypes/ComplexLeaf.cs

@@ -66 +66 @@
       if (t == null) throw new ArgumentException("No RegressionSolution was provided by the algorithm");
       return t.Model;
     }
-
     public int MinLeafSize(IRegressionProblemData pd) {
       return 3;
     }

LeafTypes/LinearLeaf.cs

@@ -49 +49 @@
       if (pd.Dataset.Rows < MinLeafSize(pd)) throw new ArgumentException("The number of training instances is too small to create a linear model");
       double rmse, cvRmse;
       noParameters = pd.AllowedInputVariables.Count() + 1;
-      return PreconstructedLinearModel.CreateConfidenceLinearModel(pd, out rmse, out cvRmse);
+      var res = PreconstructedLinearModel.CreateConfidenceLinearModel(pd, out rmse, out cvRmse);
+      return res;
     }
 
     public int MinLeafSize(IRegressionProblemData pd) {
-      return pd.AllowedInputVariables.Count() + 2;
+      return pd.AllowedInputVariables.Count() == 1 ? 2 : pd.AllowedInputVariables.Count() + 2;
     }
     #endregion
   }

LeafTypes/LogisticLeaf.cs

@@ -58 +58 @@
       get { return true; }
     }
     public IRegressionModel Build(IRegressionProblemData pd, IRandom random, CancellationToken cancellationToken, out int noParameters) {
-      if (pd.Dataset.Rows < MinLeafSize(pd)) throw new ArgumentException("The number of training instances is too small to create a linear model");
-      double rmse, cvRmse;
-      noParameters = pd.AllowedInputVariables.Count() + 1;
-      return new DampenedLinearModel(PreconstructedLinearModel.CreateConfidenceLinearModel(pd, out rmse, out cvRmse), pd, Dampening);
+      var res = (IConfidenceRegressionModel)new LinearLeaf().Build(pd, random, cancellationToken, out noParameters);
+      return new DampenedLinearModel(res, pd, Dampening);
     }
 
     public int MinLeafSize(IRegressionProblemData pd) {

LeafTypes/M5Leaf.cs

@@ +1 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item("M5Leaf", "A leaf type that uses linear models as leaf models. This is the standard for M5' regression")]
+  public class M5Leaf : ParameterizedNamedItem, ILeafModel {
+    #region Constructors & Cloning
+    [StorableConstructor]
+    private M5Leaf(bool deserializing) : base(deserializing) { }
+    private M5Leaf(M5Leaf original, Cloner cloner) : base(original, cloner) { }
+    public M5Leaf() { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new M5Leaf(this, cloner);
+    }
+    #endregion
+
+    #region IModelType
+    public bool ProvidesConfidence {
+      get { return false; }
+    }
+    public IRegressionModel Build(IRegressionProblemData pd, IRandom random, CancellationToken cancellationToken, out int noParameters) {
+      if (pd.Dataset.Rows == 0) throw new ArgumentException("The number of training instances is too small to create an M5 leaf model");
+
+      if (pd.Dataset.Rows == 1)
+        return new ConstantLeaf().Build(pd, random, cancellationToken, out noParameters);
+
+      var means = pd.AllowedInputVariables.ToDictionary(n => n, n => pd.Dataset.GetDoubleValues(n, pd.TrainingIndices).Average());
+      var variances = pd.AllowedInputVariables.ToDictionary(n => n, n => pd.Dataset.GetDoubleValues(n, pd.TrainingIndices).Variance());
+      var used = pd.AllowedInputVariables.Where(v => !variances[v].IsAlmost(0.0)).ToList();
+
+      var classMean = pd.TargetVariableTrainingValues.Average();
+      var classVar = pd.TargetVariableTrainingValues.Variance();
+
+      var model = FindBestModel(variances, means, classMean, classVar, pd, used);
+      noParameters = 1 + model.Coefficients.Count;
+      return model;
+    }
+
+
+    private static PreconstructedLinearModel FindBestModel(Dictionary<string, double> variances, Dictionary<string, double> means, double cMean, double cVar, IRegressionProblemData pd, IList<string> variables) {
+      Dictionary<string, double> coeffs;
+      double intercept;
+      do {
+        coeffs = DoRegression(pd, variables, variances, means, cMean, 1.0e-8, out intercept);
+        variables = DeselectColinear(variances, coeffs, cVar, pd, variables);
+      }
+      while (coeffs.Count != variables.Count);
+      var numAtts = variables.Count;
+      var numInst = pd.TrainingIndices.Count();
+      var fullMse = CalculateSE(coeffs, intercept, pd, variables);
+      var akaike = 1.0 * (numInst - numAtts) + 2 * numAtts;
+
+      var improved = true;
+      var currentNumAttributes = numAtts;
+
+      while (improved && currentNumAttributes > 1) {
+        improved = false;
+        currentNumAttributes--;
+        // Find attribute with smallest SC
+        var candidate = variables.ToDictionary(v => v, v => Math.Abs(coeffs[v] * Math.Sqrt(variances[v] / cVar)))
+          .OrderBy(x => x.Value).Select(x => x.Key).First();
+
+        var currVariables = variables.Where(v => !v.Equals(candidate)).ToList();
+        var currentIntercept = 0.0;
+        var currentCoeffs = DoRegression(pd, currVariables, variances, means, cMean, 1.0e-8, out currentIntercept);
+        var currentMse = CalculateSE(currentCoeffs, currentIntercept, pd, currVariables);
+        var currentAkaike = currentMse / fullMse * (numInst - numAtts) + 2 * currentNumAttributes;
+
+        if (!(currentAkaike < akaike)) continue;
+        improved = true;
+        akaike = currentAkaike;
+        coeffs = currentCoeffs;
+        intercept = currentIntercept;
+        variables = currVariables;
+      }
+
+      var pd2 = new RegressionProblemData(pd.Dataset, variables, pd.TargetVariable);
+      pd2.TestPartition.End = pd.TestPartition.End;
+      pd2.TestPartition.Start = pd.TestPartition.Start;
+      pd2.TrainingPartition.End = pd.TrainingPartition.End;
+      pd2.TrainingPartition.Start = pd.TrainingPartition.Start;
+
+      return new PreconstructedLinearModel(means, variances, coeffs, intercept, pd.TargetVariable);
+    }
+
+
+    private static Dictionary<string, double> DoRegression(IRegressionProblemData pd, IList<string> variables, Dictionary<string, double> variances, Dictionary<string, double> means, double cmean, double ridge, out double intercept) {
+      //if (pd.TrainingIndices.Count() > variables.Count) {
+      //  var pd2 = new RegressionProblemData(pd.Dataset, variables, pd.TargetVariable);
+      //  pd2.TestPartition.End = pd.TestPartition.End;
+      //  pd2.TestPartition.Start = pd.TestPartition.Start;
+      //  pd2.TrainingPartition.End = pd.TrainingPartition.End;
+      //  pd2.TrainingPartition.Start = pd.TrainingPartition.Start;
+      //
+      //  double x1, x2;
+      //  var lm = PreconstructedLinearModel.CreateConfidenceLinearModel(pd2, out x1, out x2);
+      //  intercept = lm.Intercept;
+      //  return lm.Coefficients;
+
+      var n = variables.Count;
+      var m = pd.TrainingIndices.Count();
+
+      var inTr = new double[n, m];
+      for (var i = 0; i < n; i++) {
+        var v = variables[i];
+        var vdata = pd.Dataset.GetDoubleValues(v, pd.TrainingIndices).ToArray();
+        var sd = Math.Sqrt(variances[v]);
+        var mean = means[v];
+        for (var j = 0; j < m; j++) {
+          inTr[i, j] = (vdata[j] - mean) / sd;
+        }
+      }
+
+      var y = new double[m, 1];
+      var ydata = pd.TargetVariableTrainingValues.ToArray();
+      for (var i = 0; i < m; i++)
+        y[i, 0] = ydata[i]; //no scaling for targets;
+
+
+      var aTy = new double[n, 1];
+      alglib.rmatrixgemm(n, 1, m, 1, inTr, 0, 0, 0, y, 0, 0, 0, 0, ref aTy, 0, 0); //aTy = inTr * y;
+      var aTa = new double[n, n];
+      alglib.rmatrixgemm(n, n, m, 1, inTr, 0, 0, 0, inTr, 0, 0, 1, 0, ref aTa, 0, 0); //aTa = inTr * t(inTr) +aTa //
+
+      var aTaDecomp = new double[n, n];
+      bool success;
+      var tries = 0;
+      double[] coefficients = null;
+      do {
+        for (var i = 0; i < n; i++) aTa[i, i] += ridge; // add ridge to diagonal to enforce singularity
+        try {
+          //solve "aTa * coefficients = aTy" for coefficients;
+          Array.Copy(aTa, 0, aTaDecomp, 0, aTa.Length);
+          alglib.spdmatrixcholesky(ref aTaDecomp, n, true);
+          int info;
+          alglib.densesolverreport report;
+          alglib.spdmatrixcholeskysolve(aTaDecomp, n, true, ydata, out info, out report, out coefficients);
+
+          if (info != 1) throw new Exception();
+          success = true;
+        }
+        catch (Exception) {
+          for (var i = 0; i < n; i++) aTa[i, i] -= ridge;
+          ridge *= 10; // increase ridge;
+          success = false;
+        }
+        finally {
+          tries++;
+        }
+      }
+      while (!success && tries < 100);
+      if (coefficients == null) throw new ArgumentException("No linear model could be built");
+
+      intercept = cmean;
+      var res = new Dictionary<string, double>();
+      for (var i = 0; i < n; i++) {
+        var v = variables[i];
+        res.Add(v, coefficients[i] /= Math.Sqrt(variances[v]));
+        intercept -= coefficients[i] * means[v];
+      }
+
+      return res;
+    }
+
+    private static IList<string> DeselectColinear(Dictionary<string, double> variances, Dictionary<string, double> coeffs, double cVar, IRegressionProblemData pd, IList<string> variables) {
+      var candidates = variables.ToDictionary(v => v, v => Math.Abs(coeffs[v] * Math.Sqrt(variances[v] / cVar))).Where(x => x.Value > 1.5).OrderBy(x => -x.Value).ToList();
+      if (candidates.Count == 0) return variables;
+      var c = candidates.First().Key;
+      return variables.Where(v => !v.Equals(c)).ToList();
+    }
+    private static double CalculateSE(Dictionary<string, double> coefficients, double intercept, IRegressionProblemData pd, IList<string> variables) {
+      return pd.TrainingIndices.Select(i => RegressionPrediction(i, pd, variables, coefficients, intercept) - pd.Dataset.GetDoubleValue(pd.TargetVariable, i)).Select(error => error * error).Sum();
+    }
+    private static double RegressionPrediction(int i, IRegressionProblemData pd, IList<string> variables, Dictionary<string, double> coefficients, double intercept) {
+      return intercept + variables.Sum(v => pd.Dataset.GetDoubleValue(v, i) * coefficients[v]);
+    }
+    public int MinLeafSize(IRegressionProblemData pd) {
+      return 1;
+    }
+    #endregion
+  }
+}
+ No newline at end of file

LeafTypes/M5regLeaf.cs

@@ +1 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Algorithms.DataAnalysis.Glmnet;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item("M5regLeaf", "A leaf type that uses linear models as leaf models. This is the standard for M5' regression")]
+  public class M5regLeaf : ParameterizedNamedItem, ILeafModel {
+    #region Constructors & Cloning
+    [StorableConstructor]
+    private M5regLeaf(bool deserializing) : base(deserializing) { }
+    private M5regLeaf(M5regLeaf original, Cloner cloner) : base(original, cloner) { }
+    public M5regLeaf() { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new M5regLeaf(this, cloner);
+    }
+    #endregion
+
+    #region IModelType
+    public bool ProvidesConfidence {
+      get { return true; }
+    }
+    public IRegressionModel Build(IRegressionProblemData pd, IRandom random, CancellationToken cancellationToken, out int noParameters) {
+      if (pd.Dataset.Rows < MinLeafSize(pd)) throw new ArgumentException("The number of training instances is too small to create a linear model");
+      double rmse, cvRmse;
+      noParameters = pd.AllowedInputVariables.Count() + 1;
+
+      double x1, x2;
+      var coeffs = ElasticNetLinearRegression.CalculateModelCoefficients(pd, 1, 0.2, out x1, out x2);
+      noParameters = coeffs.Length;
+      return ElasticNetLinearRegression.CreateSymbolicSolution(coeffs, pd).Model;
+    }
+
+    public int MinLeafSize(IRegressionProblemData pd) {
+      return pd.AllowedInputVariables.Count() + 2;
+    }
+    #endregion
+  }
+}
+ No newline at end of file

M5Regression.cs

@@ -21 +21 @@
     #region Parametername
     private const string GenerateRulesParameterName = "GenerateRules";
     private const string HoldoutSizeParameterName = "HoldoutSize";
-    private const string SpliterParameterName = "Spliter";
+    private const string SpliterParameterName = "Splitter";
     private const string MinimalNodeSizeParameterName = "MinimalNodeSize";
     private const string LeafModelParameterName = "LeafModel";
     private const string PruningTypeParameterName = "PruningType";
@@ -37 +37 @@
     public IFixedValueParameter<PercentValue> HoldoutSizeParameter {
       get { return (IFixedValueParameter<PercentValue>)Parameters[HoldoutSizeParameterName]; }
     }
-    public IConstrainedValueParameter<ISpliter> ImpurityParameter {
-      get { return (IConstrainedValueParameter<ISpliter>)Parameters[SpliterParameterName]; }
+    public IConstrainedValueParameter<ISplitter> ImpurityParameter {
+      get { return (IConstrainedValueParameter<ISplitter>)Parameters[SpliterParameterName]; }
     }
     public IFixedValueParameter<IntValue> MinimalNodeSizeParameter {
       get { return (IFixedValueParameter<IntValue>)Parameters[MinimalNodeSizeParameterName]; }
@@ -67 +67 @@
     public double HoldoutSize {
       get { return HoldoutSizeParameter.Value.Value; }
     }
-    public ISpliter Split {
+    public ISplitter Split {
       get { return ImpurityParameter.Value; }
     }
     public int MinimalNodeSize {
@@ -97 +97 @@
     public M5Regression() {
       var modelSet = new ItemSet<ILeafModel>(ApplicationManager.Manager.GetInstances<ILeafModel>());
       var pruningSet = new ItemSet<IPruning>(ApplicationManager.Manager.GetInstances<IPruning>());
-      var impuritySet = new ItemSet<ISpliter>(ApplicationManager.Manager.GetInstances<ISpliter>());
+      var impuritySet = new ItemSet<ISplitter>(ApplicationManager.Manager.GetInstances<ISplitter>());
       Parameters.Add(new FixedValueParameter<BoolValue>(GenerateRulesParameterName, "Whether a set of rules or a decision tree shall be created", new BoolValue(false)));
       Parameters.Add(new FixedValueParameter<PercentValue>(HoldoutSizeParameterName, "How much of the training set shall be reserved for pruning", new PercentValue(0.2)));
-      Parameters.Add(new ConstrainedValueParameter<ISpliter>(SpliterParameterName, "The type of split function used to create node splits", impuritySet, impuritySet.OfType<M5Spliter>().First()));
+      Parameters.Add(new ConstrainedValueParameter<ISplitter>(SpliterParameterName, "The type of split function used to create node splits", impuritySet, impuritySet.OfType<M5Splitter>().First()));
       Parameters.Add(new FixedValueParameter<IntValue>(MinimalNodeSizeParameterName, "The minimal number of samples in a leaf node", new IntValue(1)));
       Parameters.Add(new ConstrainedValueParameter<ILeafModel>(LeafModelParameterName, "The type of model used for the nodes", modelSet, modelSet.OfType<LinearLeaf>().First()));
       Parameters.Add(new ConstrainedValueParameter<IPruning>(PruningTypeParameterName, "The type of pruning used", pruningSet, pruningSet.OfType<M5LinearBottomUpPruning>().First()));
@@ -119 +119 @@
       if (SetSeedRandomly) SeedParameter.Value.Value = new System.Random().Next();
       random.Reset(Seed);
       var solution = CreateM5RegressionSolution(Problem.ProblemData, random, LeafModel, Split, Pruning, UseHoldout, HoldoutSize, MinimalNodeSize, GenerateRules, Results, cancellationToken);
-      AnalyzeSolution(solution);
+      AnalyzeSolution(solution, Results, Problem.ProblemData);
     }
 
     #region Static Interface
     public static IRegressionSolution CreateM5RegressionSolution(IRegressionProblemData problemData, IRandom random,
-      ILeafModel leafModel = null, ISpliter spliter = null, IPruning pruning = null,
+      ILeafModel leafModel = null, ISplitter splitter = null, IPruning pruning = null,
       bool useHoldout = false, double holdoutSize = 0.2, int minNumInstances = 4, bool generateRules = false, ResultCollection results = null, CancellationToken? cancellationToken = null) {
       //set default values
       if (leafModel == null) leafModel = new LinearLeaf();
-      if (spliter == null) spliter = new M5Spliter();
+      if (splitter == null) splitter = new M5Splitter();
       if (cancellationToken == null) cancellationToken = CancellationToken.None;
       if (pruning == null) pruning = new M5LeafBottomUpPruning();
 
+      //reduce RegressionProblemData to AllowedInput & Target column wise and to TrainingSet row wise
       var doubleVars = new HashSet<string>(problemData.Dataset.DoubleVariables);
       var vars = problemData.AllowedInputVariables.Concat(new[] {problemData.TargetVariable}).ToArray();
       if (vars.Any(v => !doubleVars.Contains(v))) throw new NotSupportedException("M5 regression supports only double valued input or output features.");
-
-      var values = vars.Select(v => problemData.Dataset.GetDoubleValues(v, problemData.TrainingIndices).ToArray()).ToArray();
-      if (values.Any(v => v.Any(x => double.IsNaN(x) || double.IsInfinity(x))))
+      var doubles = vars.Select(v => problemData.Dataset.GetDoubleValues(v, problemData.TrainingIndices).ToArray()).ToArray();
+      if (doubles.Any(v => v.Any(x => double.IsNaN(x) || double.IsInfinity(x))))
         throw new NotSupportedException("M5 regression does not support NaN or infinity values in the input dataset.");
-
-      var trainingData = new Dataset(vars, values);
+      var trainingData = new Dataset(vars, doubles);
       var pd = new RegressionProblemData(trainingData, problemData.AllowedInputVariables, problemData.TargetVariable);
       pd.TrainingPartition.End = pd.TestPartition.Start = pd.TestPartition.End = pd.Dataset.Rows;
       pd.TrainingPartition.Start = 0;
 
-      //create & build Model
-      var m5Params = new M5Parameters(pruning, minNumInstances, leafModel, pd, random, spliter, results);
-
+      //intialize M5Parameters and pruning set
+      var m5Params = new M5Parameters(pruning, minNumInstances, leafModel, pd, random, splitter, results);
       IReadOnlyList<int> trainingRows, pruningRows;
       GeneratePruningSet(problemData.TrainingIndices.ToArray(), random, useHoldout, holdoutSize, out trainingRows, out pruningRows);
 
+      //create & build Model
       IM5Model model;
-      if (generateRules)
-        model = M5RuleSetModel.CreateRuleModel(problemData.TargetVariable, m5Params);
-      else
-        model = M5TreeModel.CreateTreeModel(problemData.TargetVariable, m5Params);
+      if (generateRules) model = M5RuleSetModel.CreateRuleModel(problemData.TargetVariable, m5Params);
+      else model = M5TreeModel.CreateTreeModel(problemData.TargetVariable, m5Params);
+      model.Build(trainingRows, pruningRows, m5Params, cancellationToken.Value);
 
-      model.Build(trainingRows, pruningRows, m5Params, cancellationToken.Value);
       return model.CreateRegressionSolution(problemData);
     }
 
-    public static void UpdateM5Model(IRegressionModel model, IRegressionProblemData problemData, IRandom random,
-      ILeafModel leafModel, CancellationToken? cancellationToken = null) {
-      var m5Model = model as IM5Model;
-      if (m5Model == null) throw new ArgumentException("This type of model can not be updated");
-      UpdateM5Model(m5Model, problemData, random, leafModel, cancellationToken);
-    }
-
-    private static void UpdateM5Model(IM5Model model, IRegressionProblemData problemData, IRandom random,
-      ILeafModel leafModel = null, CancellationToken? cancellationToken = null) {
+    public static void UpdateM5Model(IM5Model model, IRegressionProblemData problemData, IRandom random, ILeafModel leafModel, CancellationToken? cancellationToken = null) {
       if (cancellationToken == null) cancellationToken = CancellationToken.None;
       var m5Params = new M5Parameters(leafModel, problemData, random);
       model.Update(problemData.TrainingIndices.ToList(), m5Params, cancellationToken.Value);
@@ -189 +178 @@
       training = perm.Take(cut).Select(i => allrows[i]).ToArray();
     }
 
-    private void AnalyzeSolution(IRegressionSolution solution) {
+    private static void AnalyzeSolution(IRegressionSolution solution, ResultCollection Results, IRegressionProblemData problemData) {
       Results.Add(new Result("RegressionSolution", (IItem)solution.Clone()));
 
-      Dictionary<string, int> frequencies;
-      if (!GenerateRules) {
-        Results.Add(M5Analyzer.CreateLeafDepthHistogram((M5TreeModel)solution.Model));
-        frequencies = M5Analyzer.GetTreeVariableFrequences((M5TreeModel)solution.Model);
+      Dictionary<string, int> frequencies = null;
+
+      var tree = solution.Model as M5TreeModel;
+      if (tree != null) {
+        Results.Add(M5Analyzer.CreateLeafDepthHistogram(tree));
+        frequencies = M5Analyzer.GetTreeVariableFrequences(tree);
+        M5Analyzer.AnalyzeNodes(tree, Results, problemData);
       }
-      else {
-        Results.Add(M5Analyzer.CreateRulesResult((M5RuleSetModel)solution.Model, Problem.ProblemData, "M5TreeResult", true));
-        frequencies = M5Analyzer.GetRuleVariableFrequences((M5RuleSetModel)solution.Model);
-        Results.Add(M5Analyzer.CreateCoverageDiagram((M5RuleSetModel)solution.Model, Problem.ProblemData));
+
+      var ruleSet = solution.Model as M5RuleSetModel;
+      if (ruleSet != null) {
+        Results.Add(M5Analyzer.CreateRulesResult(ruleSet, problemData, "M5Rules", true));
+        frequencies = M5Analyzer.GetRuleVariableFrequences(ruleSet);
+        Results.Add(M5Analyzer.CreateCoverageDiagram(ruleSet, problemData));
       }
 
       //Variable frequencies
-      var sum = frequencies.Values.Sum();
-      sum = sum == 0 ? 1 : sum;
-      var impactArray = new DoubleArray(frequencies.Select(i => (double)i.Value / sum).ToArray()) {
-        ElementNames = frequencies.Select(i => i.Key)
-      };
-      Results.Add(new Result("Variable Frequences", "relative frequencies of variables in rules and tree nodes", impactArray));
+      if (frequencies != null) {
+        var sum = frequencies.Values.Sum();
+        sum = sum == 0 ? 1 : sum;
+        var impactArray = new DoubleArray(frequencies.Select(i => (double)i.Value / sum).ToArray()) {
+          ElementNames = frequencies.Select(i => i.Key)
+        };
+        Results.Add(new Result("Variable Frequences", "relative frequencies of variables in rules and tree nodes", impactArray));
+      }
     }
     #endregion
   }

M5Utilities/M5Analyzer.cs

@@ -22 +22 @@
 using System.Collections.Generic;
 using System.Linq;
 using HeuristicLab.Analysis;
+using HeuristicLab.Common;
 using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
 using HeuristicLab.Optimization;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
-  internal static class M5Analyzer {
+  public static class M5Analyzer {
     private const string ConditionResultName = "Condition";
     private const string CoverResultName = "Covered Instances";
     private const string CoverageDiagramResultName = "Coverage";
@@ -52 +56 @@
     public static Result CreateLeafDepthHistogram(M5TreeModel treeModel) {
       var list = new List<int>();
       GetLeafDepths(treeModel.Root, 0, list);
-      var row = new DataRow("Depths", "", list.Select(x => (double) x)) {
+      var row = new DataRow("Depths", "", list.Select(x => (double)x)) {
         VisualProperties = {ChartType = DataRowVisualProperties.DataRowChartType.Histogram}
       };
       var hist = new DataTable("LeafDepths");
@@ -130 +134 @@
           }
       return res;
     }
+
+    public static void AnalyzeNodes(M5TreeModel tree, ResultCollection results, IRegressionProblemData pd) {
+      var dict = new Dictionary<int, M5NodeModel>();
+      var modelNumber = new IntValue(1);
+      var symtree = new SymbolicExpressionTree(MirrorTree(tree.Root, dict, modelNumber, pd.Dataset, pd.TrainingIndices.ToList()));
+      results.AddOrUpdateResult("DecisionTree", symtree);
+
+      if (dict.Count > 200) return;
+      var models = new ResultCollection();
+      results.AddOrUpdateResult("NodeModels", models);
+      foreach (var m in dict.Keys.OrderBy(x => x)) {
+        models.AddOrUpdateResult("Model " + m, dict[m].CreateRegressionSolution(pd));
+      }
+    }
+
+    private static SymbolicExpressionTreeNode MirrorTree(M5NodeModel node, IDictionary<int, M5NodeModel> dict, IntValue nextId, IDataset data, IReadOnlyList<int> rows) {
+      if (node.IsLeaf) {
+        var i = nextId.Value++;
+        dict.Add(i, node);
+        return new SymbolicExpressionTreeNode(new TextSymbol("Model " + i + " " + rows.Count + " Instances"));
+      }
+
+      var text = node.SplitAttribute + " <= " + node.SplitValue.ToString("0.###") + " pf= " + node.PruningStrength.ToString("0.###");
+      var textNode = new SymbolicExpressionTreeNode(new TextSymbol(text));
+      IReadOnlyList<int> lrows, rrows;
+      M5StaticUtilities.SplitRows(rows, data, node.SplitAttribute, node.SplitValue, out lrows, out rrows);
+
+      textNode.AddSubtree(MirrorTree(node.Left, dict, nextId, data, lrows));
+      textNode.AddSubtree(MirrorTree(node.Right, dict, nextId, data, rrows));
+
+      return textNode;
+    }
+
+
+    [StorableClass]
+    private class TextSymbol : Symbol {
+      [StorableConstructor]
+      public TextSymbol(bool deserializing) : base(deserializing) { }
+      public TextSymbol(Symbol original, Cloner cloner) : base(original, cloner) { }
+      public TextSymbol(string name) : base(name, "") {
+        this.Name = name;
+      }
+      public override IDeepCloneable Clone(Cloner cloner) {
+        return new TextSymbol(this, cloner);
+      }
+      public override int MinimumArity {
+        get { return 0; }
+      }
+      public override int MaximumArity {
+        get { return int.MaxValue; }
+      }
+    }
   }
 }
+ No newline at end of file

M5Utilities/M5Parameters.cs

@@ -26 +26 @@
 using HeuristicLab.Problems.DataAnalysis;
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
-  internal class M5Parameters {
-    private readonly ISpliter splitter;
+  public class M5Parameters {
+    private readonly ISplitter splitter;
     private readonly IPruning pruning;
     private readonly ILeafModel leafModel;
     private readonly int minLeafSize;
@@ -34 +34 @@
     private readonly IRegressionProblemData problemData;
     private readonly IRandom random;
     private readonly ResultCollection results;
-    public ISpliter Spliter {
+    public ISplitter Splitter {
       get { return splitter; }
     }
     public IPruning Pruning {
@@ -66 +66 @@
     }
 
     public M5Parameters(IPruning pruning, int minleafSize, ILeafModel leafModel,
-      IRegressionProblemData problemData, IRandom random, ISpliter splitter, ResultCollection results) {
+      IRegressionProblemData problemData, IRandom random, ISplitter splitter, ResultCollection results) {
       this.problemData = problemData;
       this.random = random;
       this.leafModel = leafModel;

M5Utilities/M5StaticUtilities.cs

@@ -42 +42 @@
       }
       if (alg.ExecutionState != ExecutionState.Paused) alg.Prepare();
       alg.Start(cancellationToken);
-      return alg.Results;
+      var res = alg.Results;
+      alg.Runs.Clear();
+      return res;
     }
 
     public static void SplitRows(IReadOnlyList<int> rows, IDataset data, string splitAttr, double splitValue, out IReadOnlyList<int> leftRows, out IReadOnlyList<int> rightRows) {
-      //TODO check and revert points at borders are now used multipe times
+      //TODO check and revert?: points at borders are now used multipe times
       var assignment = data.GetDoubleValues(splitAttr, rows).Select(x => x.IsAlmost(splitValue) ? 2 : x < splitValue ? 0 : 1).ToArray();
       leftRows = rows.Zip(assignment, (i, b) => new {i, b}).Where(x => x.b == 0 || x.b == 2).Select(x => x.i).ToList();
       rightRows = rows.Zip(assignment, (i, b) => new {i, b}).Where(x => x.b > 0).Select(x => x.i).ToList();

MetaModels/M5NodeModel.cs

@@ -31 +31 @@
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
   [StorableClass]
-  internal class M5NodeModel : RegressionModel {
+  public class M5NodeModel : RegressionModel {
     #region Properties
+    public double PruningStrength;
+
     [Storable]
     internal bool IsLeaf { get; private set; }
     [Storable]
@@ -106 +108 @@
       SplitValue = double.NaN;
       string attr;
       double splitValue;
-      IsLeaf = !m5Params.Spliter.Split(new RegressionProblemData(M5StaticUtilities.ReduceDataset(m5Params.Data, rows, variables, TargetVariable), variables, TargetVariable), m5Params.MinLeafSize, out attr, out splitValue);
+      IsLeaf = !m5Params.Splitter.Split(new RegressionProblemData(M5StaticUtilities.ReduceDataset(m5Params.Data, rows, variables, TargetVariable), variables, TargetVariable), m5Params.MinLeafSize, out attr, out splitValue);
       if (IsLeaf) return;
 
       //split Dataset

MetaModels/M5RuleModel.cs

@@ -31 +31 @@
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
   [StorableClass]
-  internal class M5RuleModel : RegressionModel {
+  public class M5RuleModel : RegressionModel {
     #region Properties
     [Storable]
     internal string[] SplitAttributes { get; private set; }

MetaModels/M5RuleSetModel.cs

@@ -31 +31 @@
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
   [StorableClass]
-  internal class M5RuleSetModel : RegressionModel, IM5Model {
+  public class M5RuleSetModel : RegressionModel, IM5Model {
     private const string NumRulesResultName = "Number of rules";
     private const string CoveredInstancesResultName = "Covered instances";
 

MetaModels/M5TreeModel.cs

@@ -31 +31 @@
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
   [StorableClass]
-  internal class M5TreeModel : RegressionModel, IM5Model {
+  public class M5TreeModel : RegressionModel, IM5Model {
     public const string NumCurrentLeafsResultName = "Number of current leafs";
     #region Properties
     [Storable]
@@ -69 +69 @@
 
     #region IM5Model
     public void Build(IReadOnlyList<int> trainingRows, IReadOnlyList<int> pruningRows, M5Parameters m5Params, CancellationToken cancellationToken) {
+      //create intial (overfitted tree)
       Root = M5NodeModel.CreateNode(m5Params.TargetVariable, m5Params);
       Root.Split(trainingRows, m5Params);
 
-      InitializeLeafCounter(m5Params);
+      //intitalize leafs counter
+      var leafs = Root.EnumerateNodes().Count(x => x.IsLeaf);
+      if (!m5Params.Results.ContainsKey(NumCurrentLeafsResultName))
+        m5Params.Results.Add(new Result(NumCurrentLeafsResultName, new IntValue(leafs)));
+      else ((IntValue)m5Params.Results[NumCurrentLeafsResultName].Value).Value = leafs;
 
-      var buPruner = m5Params.Pruning as BottomUpPruningBase;
-      if (buPruner != null) buPruner.Prune(this, trainingRows, pruningRows, m5Params, cancellationToken);
+      //prune
+      m5Params.Pruning.Prune(this, trainingRows, pruningRows, m5Params, cancellationToken);
 
+      //build final leaf models
       Root.BuildLeafModels(trainingRows.Union(pruningRows).ToArray(), m5Params, cancellationToken);
     }
 
@@ -85 +91 @@
     }
     #endregion
 
-    #region Helpers
-    private void InitializeLeafCounter(M5Parameters m5Params) {
-      if (!m5Params.Results.ContainsKey(NumCurrentLeafsResultName))
-        m5Params.Results.Add(new Result(NumCurrentLeafsResultName, new IntValue(Root.EnumerateNodes().Count(x => x.IsLeaf))));
-      else ((IntValue)m5Params.Results[NumCurrentLeafsResultName].Value).Value = Root.EnumerateNodes().Count(x => x.IsLeaf);
-    }
-    #endregion
-
     [StorableClass]
     private class ConfidenceM5TreeModel : M5TreeModel, IConfidenceRegressionModel {
       #region HLConstructors & Cloning

Pruning/BottomUpPruningBase.cs

@@ -19 +19 @@
  */
 #endregion
 
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
@@ -46 +47 @@
     protected BottomUpPruningBase(bool deserializing) : base(deserializing) { }
     protected BottomUpPruningBase(BottomUpPruningBase original, Cloner cloner) : base(original, cloner) { }
     protected BottomUpPruningBase() {
-      Parameters.Add(new FixedValueParameter<DoubleValue>(PruningStrengthParameterName, "The strength of the pruning. Higher values force the algorithm to create simpler models", new DoubleValue(4.0)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(PruningStrengthParameterName, "The strength of the pruning. Higher values force the algorithm to create simpler models", new DoubleValue(2.0)));
     }
     #endregion
 
@@ -58 +59 @@
     }
     #endregion
 
-    internal void Prune(M5TreeModel treeModel, IReadOnlyList<int> trainingRows, IReadOnlyList<int> pruningRows, M5Parameters m5Params, CancellationToken cancellationToken) {
-      var globalStdDev = m5Params.Data.GetDoubleValues(m5Params.TargetVariable, trainingRows).StandardDeviationPop();
-
-      Prune(treeModel.Root, trainingRows, pruningRows, m5Params, new Dictionary<M5NodeModel, int>(), new Dictionary<M5NodeModel, int>(), cancellationToken, globalStdDev);
+    public void Prune(M5TreeModel treeModel, IReadOnlyList<int> trainingRows, IReadOnlyList<int> pruningRows, M5Parameters m5Params, CancellationToken cancellationToken) {
+      Prune(treeModel.Root, trainingRows, pruningRows, m5Params, new Dictionary<M5NodeModel, int>(), new Dictionary<M5NodeModel, int>(), cancellationToken);
     }
 
     private bool Prune(M5NodeModel node, IReadOnlyList<int> trainingRows, IReadOnlyList<int> pruningRows, M5Parameters m5Params,
       Dictionary<M5NodeModel, int> modelComplexities, Dictionary<M5NodeModel, int> nodeComplexities,
-      CancellationToken cancellationToken, double globalStdDev) {
+      CancellationToken cancellationToken) {
       //build pruning model
       int numModelParams;
       var pruningModel = M5StaticUtilities.BuildModel(trainingRows, m5Params, PruningLeafModel(m5Params.LeafModel), cancellationToken, out numModelParams);
@@ -79 +78 @@
       }
 
       //split training & pruning data
-      IReadOnlyList<int> leftTest, rightTest;
-      M5StaticUtilities.SplitRows(pruningRows, m5Params.Data, node.SplitAttribute, node.SplitValue, out leftTest, out rightTest);
+      IReadOnlyList<int> leftPruning, rightPruning;
+      M5StaticUtilities.SplitRows(pruningRows, m5Params.Data, node.SplitAttribute, node.SplitValue, out leftPruning, out rightPruning);
       IReadOnlyList<int> leftTraining, rightTraining;
       M5StaticUtilities.SplitRows(trainingRows, m5Params.Data, node.SplitAttribute, node.SplitValue, out leftTraining, out rightTraining);
 
       //prune children frist
-      var lpruned = Prune(node.Left, leftTraining, leftTest, m5Params, modelComplexities, nodeComplexities, cancellationToken, globalStdDev);
-      var rpruned = Prune(node.Right, rightTraining, rightTest, m5Params, modelComplexities, nodeComplexities, cancellationToken, globalStdDev);
+      var lpruned = Prune(node.Left, leftTraining, leftPruning, m5Params, modelComplexities, nodeComplexities, cancellationToken);
+      var rpruned = Prune(node.Right, rightTraining, rightPruning, m5Params, modelComplexities, nodeComplexities, cancellationToken);
       nodeComplexities.Add(node, nodeComplexities[node.Left] + nodeComplexities[node.Right] + 1);
 
       //TODO check if this reduces quality. It reduces training effort (consideraby for some pruningTypes)
@@ -93 +92 @@
       if (!lpruned && !rpruned) return false;
 
       //check if pruning will happen on this node
-      if (!DecidePruneNode(node, m5Params, pruningRows, modelComplexities, nodeComplexities, globalStdDev)) return false;
+      if (!DecidePruneNode(node, m5Params, pruningRows, modelComplexities, nodeComplexities)) return false;
 
       //convert to leafNode
       ((IntValue)m5Params.Results[M5TreeModel.NumCurrentLeafsResultName].Value).Value -= node.EnumerateNodes().Count(x => x.IsLeaf) - 1;
 
-      //TODO chack wether removal is beneficial 
       nodeComplexities.Remove(node.Left);
       nodeComplexities.Remove(node.Right);
       modelComplexities.Remove(node.Left);
@@ -110 +108 @@
     }
 
     private bool DecidePruneNode(M5NodeModel node, M5Parameters m5Params, IReadOnlyCollection<int> testRows,
-      IReadOnlyDictionary<M5NodeModel, int> modelComplexities, IReadOnlyDictionary<M5NodeModel, int> nodeComplexities,
-      double globalStdDev) {
+      IReadOnlyDictionary<M5NodeModel, int> modelComplexities, IReadOnlyDictionary<M5NodeModel, int> nodeComplexities) {
       if (testRows.Count == 0) return true;
 
       //create regressionProblemdata from pruning data 
@@ -123 +120 @@
 
       //evaluate combined sub nodes and pruning model
       var rmsModel = node.Model.CreateRegressionSolution(pd).TestRootMeanSquaredError;
+      rmsModel = rmsModel.IsAlmost(0.0) ? 0 : rmsModel;
       var rmsSubTree = node.CreateRegressionSolution(pd).TestRootMeanSquaredError;
+      rmsSubTree = rmsSubTree.IsAlmost(0.0) ? 0 : rmsSubTree;
 
+      int rows = pd.Dataset.Rows;
+      node.PruningStrength = rows * (rmsModel * (rows - nodeComplexities[node]) + rmsSubTree * (modelComplexities[node] - rows));
+      node.PruningStrength /= rmsModel * modelComplexities[node] * (nodeComplexities[node] - rows) + rmsSubTree * nodeComplexities[node] * (rows - modelComplexities[node]);
+
+      var pf1 = PruningFactor(pd.Dataset.Rows, modelComplexities[node]);
+      var pf2 = PruningFactor(pd.Dataset.Rows, nodeComplexities[node]);
       //weigh, compare and decide
-      var adjustedRmsModel = rmsModel * PruningFactor(pd.Dataset.Rows, modelComplexities[node]);
-      var adjustedRmsTree = rmsSubTree * PruningFactor(pd.Dataset.Rows, nodeComplexities[node.Left] + nodeComplexities[node.Right] + 1);
-      return adjustedRmsModel <= adjustedRmsTree;
+      var adjustedRmsModel = rmsModel * pf1;
+      var adjustedRmsTree = rmsSubTree * pf2;
+
+      return adjustedRmsModel < adjustedRmsTree;
     }
 
     private double PruningFactor(int noInstances, int noParams) {
-      return noInstances <= noParams ? 10.0 : (noInstances + PruningStrength * noParams) / (noInstances - PruningStrength * noParams);
+      //in the original M5 tree a cut off is used:
+      if (noInstances <= noParams) return 10;
+      //but to have at least some punishment for additional parameters, I would prefer: */
+      if (noInstances <= noParams) return noInstances + PruningStrength * noParams;
+
+      return (noInstances + PruningStrength * noParams) / (noInstances - noParams);
     }
   }
 }
+ No newline at end of file

Pruning/NoPruning.cs

@@ -20 +20 @@
 #endregion
 
 using System.Collections.Generic;
+using System.Threading;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
@@ -40 +41 @@
     public int MinLeafSize(IRegressionProblemData pd, ILeafModel leafModel) {
       return 0;
     }
+
+    public void Prune(M5TreeModel treeModel, IReadOnlyList<int> trainingRows, IReadOnlyList<int> pruningRows, M5Parameters m5Params, CancellationToken cancellationToken) { }
     #endregion
   }
 }
+ No newline at end of file

Spliting/CorrelationImpuritiyCalculator.cs

@@ +1 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Helper class for incremental split calculation.
+  /// Used while moving a potential Spliter along the ordered training Instances
+  /// </summary>
+  internal class CorrelationImpuritiyCalculator {
+    #region state
+    //Data
+    private readonly List<double> attributeValues;
+    private readonly List<double> targetValues;
+    private readonly double order;
+    private readonly UnivariateOnlineLR left;
+    private readonly UnivariateOnlineLR right;
+    #endregion
+
+    #region Properties
+    public double Impurity { get; private set; }
+    public double SplitValue {
+      get {
+        if (left.Size <= 0) return double.NegativeInfinity;
+        if (left.Size >= attributeValues.Count) return double.PositiveInfinity;
+        return (attributeValues[left.Size - 1] + attributeValues[left.Size]) / 2;
+      }
+    }
+    public bool ValidPosition {
+      get { return !attributeValues[left.Size - 1].IsAlmost(attributeValues[left.Size]); }
+    }
+    public int LeftSize {
+      get { return left.Size; }
+    }
+    #endregion
+
+    #region Constructors
+    public CorrelationImpuritiyCalculator(int partition, IEnumerable<double> atts, IEnumerable<double> targets, double order) {
+      if (order <= 0) throw new ArgumentException("Splitter order must be larger than 0");
+      this.order = order;
+      attributeValues = atts.ToList();
+      targetValues = targets.ToList();
+      left = new UnivariateOnlineLR(attributeValues.Take(partition).ToList(), targetValues.Take(partition).ToList());
+      right = new UnivariateOnlineLR(attributeValues.Skip(partition).ToList(), targetValues.Skip(partition).ToList());
+      UpdateImpurity();
+    }
+    #endregion
+
+    #region IImpurityCalculator
+    public void Increment() {
+      var target = targetValues[left.Size];
+      var att = attributeValues[left.Size];
+      left.Add(att, target);
+      right.Remove(att, target);
+      UpdateImpurity();
+    }
+    #endregion
+
+    private void UpdateImpurity() {
+      var yl = Math.Pow(left.Ssr, 1.0 / order);
+      var yr = Math.Pow(right.Ssr, 1.0 / order);
+      if (left.Size > 1 && right.Size > 1) Impurity = -yl - yr;
+      else Impurity = double.MinValue;
+    }
+  }
+}
+ No newline at end of file

Spliting/CorrelationSplitter.cs

@@ +1 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item("CorrelationSplitter", "An experimental split selector that uses correlation coefficients")]
+  public class CorrelationSplitter : ParameterizedNamedItem, ISplitter {
+    public const string OrderParameterName = "Order";
+    public IFixedValueParameter<DoubleValue> OrderParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[OrderParameterName]; }
+    }
+    public double Order {
+      get { return OrderParameter.Value.Value; }
+    }
+
+    #region Constructors & Cloning
+    [StorableConstructor]
+    private CorrelationSplitter(bool deserializing) { }
+    private CorrelationSplitter(CorrelationSplitter original, Cloner cloner) : base(original, cloner) { }
+    public CorrelationSplitter() {
+      Parameters.Add(new FixedValueParameter<DoubleValue>(OrderParameterName, "The exponent in the split calculation ssrLeft^(1/Order)+ssrRight^(1/Order).", new DoubleValue(1)));
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CorrelationSplitter(this, cloner);
+    }
+    #endregion
+
+    #region ISplitType
+    public bool Split(IRegressionProblemData splitData, int minLeafSize, out string splitAttr, out double splitValue) {
+      var bestSize = 0;
+      var bestImpurity = double.MinValue;
+      var bestSplitValue = 0.0;
+      var bestSplitAttr = string.Empty;
+      splitAttr = bestSplitAttr;
+      splitValue = bestSplitValue;
+      if (splitData.Dataset.Rows < minLeafSize * 2) return false;
+
+      //find best Attribute for the Splitter
+      foreach (var attr in splitData.AllowedInputVariables) {
+        int size;
+        double impurity, sValue;
+        var sortedData = splitData.Dataset.GetDoubleValues(attr).Zip(splitData.Dataset.GetDoubleValues(splitData.TargetVariable), Tuple.Create).OrderBy(x => x.Item1).ToArray();
+        AttributeSplit(sortedData.Select(x => x.Item1).ToArray(), sortedData.Select(x => x.Item2).ToArray(), minLeafSize, out size, out impurity, out sValue);
+        if (!(bestImpurity < impurity)) continue;
+        bestImpurity = impurity;
+        bestSize = size;
+        bestSplitValue = sValue;
+        bestSplitAttr = attr;
+      }
+
+      splitAttr = bestSplitAttr;
+      splitValue = bestSplitValue;
+
+      //if no suitable split exists => leafNode
+      return bestSize >= minLeafSize && bestSize <= splitData.Dataset.Rows - minLeafSize;
+    }
+
+    private void AttributeSplit(IReadOnlyList<double> attValues, IEnumerable<double> targetValues, int minLeafSize, out int leftSize, out double maxImpurity, out double splitValue) {
+      leftSize = -1;
+      splitValue = double.MinValue;
+      maxImpurity = double.NegativeInfinity;
+      var splitValues = new List<double>();
+      var splitSizes = new List<int>();
+      var length = attValues.Count;
+
+      var start = minLeafSize;
+      while (attValues[start - 1].IsAlmost(attValues[start]) && start < length)
+        start++;
+      if (start >= length) return;
+
+      var imp = new CorrelationImpuritiyCalculator(minLeafSize, attValues, targetValues, Order);
+      maxImpurity = imp.Impurity;
+      splitValues.Add(imp.SplitValue);
+      splitSizes.Add(imp.LeftSize);
+
+      while (imp.LeftSize < length - minLeafSize) {
+        imp.Increment();
+        if (!imp.ValidPosition) continue; //splits can not be made between to equal points
+
+        if (imp.Impurity.IsAlmost(maxImpurity)) {
+          splitValues.Add(imp.SplitValue);
+          splitSizes.Add(imp.LeftSize);
+          continue;
+        }
+
+        if (imp.Impurity < maxImpurity) continue;
+        splitValues.Clear();
+        splitSizes.Clear();
+        maxImpurity = imp.Impurity;
+        splitValues.Add(imp.SplitValue);
+        splitSizes.Add(imp.LeftSize);
+      }
+
+      var j = splitSizes.Count / 2;
+      if (splitSizes.Count == 0) return;
+      splitValue = splitValues[j];
+      leftSize = splitSizes[j];
+    }
+    #endregion
+  }
+}
+ No newline at end of file

Spliting/M5Splitter.cs

@@ +1 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item("M5Splitter", "A split selector that uses the ratio between Variances^(1/Order) to determine good splits")]
+  public class M5Splitter : ParameterizedNamedItem, ISplitter {
+    public const string OrderParameterName = "Order";
+    public IFixedValueParameter<DoubleValue> OrderParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[OrderParameterName]; }
+    }
+    public double Order {
+      get { return OrderParameter.Value.Value; }
+    }
+
+    #region Constructors & Cloning
+    [StorableConstructor]
+    private M5Splitter(bool deserializing) { }
+    private M5Splitter(M5Splitter original, Cloner cloner) : base(original, cloner) { }
+    public M5Splitter() {
+      Parameters.Add(new FixedValueParameter<DoubleValue>(OrderParameterName, "The exponent in the split calculation sum (x_i - x_avg)^Order.", new DoubleValue(5)));
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new M5Splitter(this, cloner);
+    }
+    #endregion
+
+    #region ISplitType
+    public bool Split(IRegressionProblemData splitData, int minLeafSize, out string splitAttr, out double splitValue) {
+      var bestPos = 0;
+      var bestImpurity = double.MinValue;
+      var bestSplitValue = 0.0;
+      var bestSplitAttr = string.Empty;
+      splitAttr = bestSplitAttr;
+      splitValue = bestSplitValue;
+      if (splitData.Dataset.Rows < minLeafSize) return false;
+      //find best Attribute for the Splitter
+      foreach (var attr in splitData.AllowedInputVariables) {
+        int pos;
+        double impurity, sValue;
+        var sortedData = splitData.Dataset.GetDoubleValues(attr).Zip(splitData.Dataset.GetDoubleValues(splitData.TargetVariable), Tuple.Create).OrderBy(x => x.Item1).ToArray();
+        AttributeSplit(sortedData.Select(x => x.Item1).ToArray(), sortedData.Select(x => x.Item2).ToArray(), minLeafSize, out pos, out impurity, out sValue);
+        if (!(bestImpurity < impurity)) continue;
+        bestImpurity = impurity;
+        bestPos = pos;
+        bestSplitValue = sValue;
+        bestSplitAttr = attr;
+      }
+
+      splitAttr = bestSplitAttr;
+      splitValue = bestSplitValue;
+      //if no suitable split exists => leafNode
+      return bestPos + 1 >= minLeafSize && bestPos <= splitData.Dataset.Rows - minLeafSize;
+    }
+
+    private void AttributeSplit(IReadOnlyList<double> attValues, IReadOnlyList<double> targetValues, int minLeafSize, out int position, out double maxImpurity, out double splitValue) {
+      position = 0;
+      maxImpurity = -1E20;
+      splitValue = 0.0;
+      var length = targetValues.Count;
+
+
+      // weka code
+      var low = 0;
+      var high = length - 1;
+      if (high - low + 1 < 4) return;
+      var len = Math.Max(minLeafSize - 1, high - low + 1 < 5 ? 1 : (high - low + 1) / 5);
+      position = low;
+      var part = low + len - 1;
+      var imp = new OrderImpurityCalculator(part + 1, targetValues, Order);
+
+
+      //if (imp.Impurity > maxImpurity && !attValues[part - 1].IsAlmost(attValues[part])) {
+      //  maxImpurity = imp.Impurity;
+      //  splitValue = (attValues[part - 1] + attValues[part]) / 2;
+      //  position = part;
+      //}
+
+      for (var i = low + len; i < high - len; i++) {
+        imp.Increment(targetValues[i], OrderImpurityCalculator.IncrementType.Left);
+        if (attValues[i].IsAlmost(attValues[i + 1])) continue; //splits can not be made between to equal points
+        if (imp.Impurity < maxImpurity) continue;
+        maxImpurity = imp.Impurity;
+        splitValue = (attValues[i] + attValues[i + 1]) / 2;
+        position = i;
+      }
+    }
+    #endregion
+  }
+}
+ No newline at end of file

Spliting/NeumaierSum.cs

@@ +1 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Helper class for incremental split calculation.
+  /// Used while moving a potential Splitter along the ordered training Instances
+  /// </summary>
+  internal class NeumaierSum {
+    #region state
+    private double sum;
+    private double correction;
+    #endregion
+
+    #region Constructors
+    public NeumaierSum(double startvalue) {
+      sum = startvalue;
+      correction = 0;
+    }
+    #endregion
+
+    [MethodImpl(MethodImplOptions.NoOptimization)]
+    public void Add(double value) {
+      var t = sum + value;
+      var absSum = sum > 0 ? sum : -sum;
+      var absv = value > 0 ? value : -value;
+      if (absSum >= absv)
+        correction += (sum - t) + value;
+      else
+        correction += (value - t) + sum;
+      sum = t;
+    }
+
+    public double Get() {
+      return sum + correction;
+    }
+
+    public void Mul(double value) {
+      sum *= value;
+      correction *= value;
+    }
+  }
+}
+ No newline at end of file

Spliting/OptimumSearchingSplitter.cs

@@ +1 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item("OptimumSearchingSplitter", "A split selector that favours higher resolution splits near percieved optima.\n Decribed in \"Model-Based Genetic Algorithms for Algorithm Configuration\" by Carlos Ansotegui et al ")]
+  public class OptimumSearchingSplitter : ParameterizedNamedItem, ISplitter {
+    public const string SearchStrengthParameterName = "Search Strength";
+    public const string MaximizationParamterName = "Maximization";
+    public const string OrderParameterName = "Order";
+    public IFixedValueParameter<DoubleValue> OrderParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[OrderParameterName]; }
+    }
+    public IFixedValueParameter<PercentValue> SearchStrengthParameter {
+      get { return (IFixedValueParameter<PercentValue>)Parameters[SearchStrengthParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> MaximizationParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[MaximizationParamterName]; }
+    }
+    public double Order {
+      get { return OrderParameter.Value.Value; }
+    }
+    public double SearchStrength {
+      get { return SearchStrengthParameter.Value.Value; }
+    }
+    public bool Maximization {
+      get { return MaximizationParameter.Value.Value; }
+    }
+
+    #region Constructors & Cloning
+    [StorableConstructor]
+    private OptimumSearchingSplitter(bool deserializing) { }
+    private OptimumSearchingSplitter(OptimumSearchingSplitter original, Cloner cloner) : base(original, cloner) { }
+    public OptimumSearchingSplitter() {
+      Parameters.Add(new FixedValueParameter<DoubleValue>(OrderParameterName, "The exponent in the split calculation sum (x_i - x_avg)^Order.", new DoubleValue(2)));
+      Parameters.Add(new FixedValueParameter<PercentValue>(SearchStrengthParameterName, "How strong the spliting process should be skewed towards/away from the percieved optimum", new PercentValue(0.10)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(MaximizationParamterName, "Whether the splitting procedure should asume a minimization or maximization procedure."));
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new OptimumSearchingSplitter(this, cloner);
+    }
+    #endregion
+
+    #region ISplitType
+    public bool Split(IRegressionProblemData splitData, int minLeafSize, out string splitAttr, out double splitValue) {
+      var bestImpurity = double.MinValue;
+      var bestSplitValue = 0.0;
+      var bestSplitAttr = string.Empty;
+      var bestSize = 0;
+      splitAttr = bestSplitAttr;
+      splitValue = bestSplitValue;
+
+      var targets = splitData.TargetVariableValues.ToArray();
+      var vh = targets.Quantile(Maximization ? 1 - SearchStrength : SearchStrength);
+      var lower = new HashSet<int>(targets.Select((x, i) => new {x, i}).Where(e => e.x < vh).Select(e => e.i));
+
+      if (splitData.Dataset.Rows < minLeafSize) return false;
+
+      foreach (var attr in splitData.AllowedInputVariables) {
+        int pos;
+        double impurity, sValue;
+        AttributeSplit(splitData, lower, attr, vh, minLeafSize, out pos, out impurity, out sValue);
+        if (!(bestImpurity < impurity)) continue;
+        bestImpurity = impurity;
+        bestSplitValue = sValue;
+        bestSplitAttr = attr;
+        bestSize = pos;
+      }
+
+      splitAttr = bestSplitAttr;
+      splitValue = bestSplitValue;
+
+      //if no suitable split exists => leafNode
+      return bestSize >= minLeafSize && bestSize <= splitData.Dataset.Rows - minLeafSize;
+    }
+
+    private void AttributeSplit(IRegressionProblemData splitData, ICollection<int> t, string attribute, double vh, int minLeafSize, out int leftSize, out double maxImpurity, out double splitValue) {
+      leftSize = 0;
+      maxImpurity = -1E20;
+      splitValue = 0.0;
+      var length = splitData.Dataset.Rows;
+
+      double lls = 0, rls = 0, lts = 0, rts = 0;
+      int ltn = 0, rtn = t.Count;
+
+      var points = splitData.Dataset.GetDoubleValues(attribute).Select((x, i) => new {x, i}).OrderBy(e => e.x).ToArray();
+      for (var i = 0; i < length - minLeafSize; i++) {
+        var point = points[i];
+        var con = Contibution(splitData, vh, point.i);
+        //move contribution to and from respcetive sums
+        if (t.Contains(i)) {
+          lls += con;
+          rls -= con;
+        }
+        else {
+          ltn++;
+          rtn--;
+          lts += con;
+          rts -= con;
+        }
+
+        //splits can not be made between to equal points
+        if (point.x.IsAlmost(points[i + 1].x)) continue;
+
+        //calculate impurity / score
+        var al = (ltn + lts) / (1 + lls);
+        var ar = (rtn + rts) / (1 + rls);
+        var impurity = ltn > rtn ? al : ltn < rtn ? ar : Math.Min(al, ar);
+
+        if (i < minLeafSize || impurity < maxImpurity) continue;
+        maxImpurity = impurity;
+        splitValue = (point.x + points[i + 1].x) / 2;
+        leftSize = i + 1;
+      }
+    }
+
+    private double Contibution(IRegressionProblemData splitData, double vh, int i) {
+      var v = splitData.Dataset.GetDoubleValue(splitData.TargetVariable, i) - vh;
+      return Math.Pow(v, Order);
+    }
+    #endregion
+  }
+}
+ No newline at end of file

Spliting/OrderImpurityCalculator.cs

@@ -27 +27 @@
 namespace HeuristicLab.Algorithms.DataAnalysis {
   /// <summary>
   /// Helper class for incremental split calculation.
-  /// Used while moving a potential Spliter along the ordered training Instances
+  /// Used while moving a potential Splitter along the ordered training Instances
   /// </summary>
   internal class OrderImpurityCalculator {
     internal enum IncrementType {
@@ -104 +104 @@
       VarLeft = NoLeft <= 0 ? 0 : Math.Abs(NoLeft * SqSumLeft - SumLeft * SumLeft) / (NoLeft * NoLeft);
       VarRight = NoRight <= 0 ? 0 : Math.Abs(NoRight * SqSumRight - SumRight * SumRight) / (NoRight * NoRight);
 
-      if (Order <= 0) throw new ArgumentException("Spliter order must be larger than 0");
+      if (Order <= 0) throw new ArgumentException("Splitter order must be larger than 0");
       if (Order.IsAlmost(1)) {
         y = VarTotal;
         yl = VarLeft;
@@ -115 +115 @@
         yl = Math.Pow(VarLeft, 1.0 / Order);
         yr = Math.Pow(VarRight, 1.0 / Order);
       }
-      var t = NoRight + NoLeft;
-      if (NoLeft <= 0.0 || NoRight <= 0.0) Impurity = double.MinValue; //Spliter = 0;
-      else Impurity = y - NoLeft / t * yl - NoRight / t * yr; //  Spliter = y - NoLeft / NoRight * yl - NoRight / NoLeft * yr
+      if (NoLeft <= 0.0 || NoRight <= 0.0) Impurity = double.MinValue; //Splitter = 0;
+      else Impurity = y - (NoLeft * yl + NoRight * yr) / (NoRight + NoLeft);
     }
     #endregion
   }

Spliting/UnivariateOnlineLR.cs

@@ +1 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Helper class for incremental split calculation.
+  /// Used while moving a potential Spliter along the ordered training Instances
+  /// </summary>
+  internal class UnivariateOnlineLR {
+    #region state
+    private readonly NeumaierSum targetMean;
+    private readonly NeumaierSum attributeMean;
+    private readonly NeumaierSum targetVarSum;
+    private readonly NeumaierSum attributeVarSum;
+    private readonly NeumaierSum comoment;
+    private readonly NeumaierSum ssr;
+    private int size;
+    #endregion
+
+    public double Ssr {
+      get { return ssr.Get(); }
+    }
+    public int Size {
+      get { return size; }
+    }
+
+    private double Beta {
+      get { return comoment.Get() / attributeVarSum.Get(); }
+    }
+
+    private double Alpha {
+      get { return targetMean.Get() - Beta * attributeMean.Get(); }
+    }
+
+    public UnivariateOnlineLR(ICollection<double> attributeValues, ICollection<double> targetValues) {
+      if (attributeValues.Count != targetValues.Count) throw new ArgumentException("Targets and Attributes need to have the same length");
+      size = attributeValues.Count;
+
+      var yMean = targetValues.Average();
+      var xMean = attributeValues.Average();
+      targetMean = new NeumaierSum(yMean);
+      attributeMean = new NeumaierSum(xMean);
+      targetVarSum = new NeumaierSum(targetValues.VariancePop() * size);
+      attributeVarSum = new NeumaierSum(attributeValues.VariancePop() * size);
+      comoment = new NeumaierSum(attributeValues.Zip(targetValues, (x, y) => (x - xMean) * (y - yMean)).Sum());
+
+      var beta = comoment.Get() / attributeVarSum.Get();
+      var alpha = yMean - beta * xMean;
+      ssr = new NeumaierSum(attributeValues.Zip(targetValues, (x, y) => y - alpha - beta * x).Sum(x => x * x));
+    }
+
+    public void Add(double attributeValue, double targetValue) {
+      var predictOld = Predict(attributeValue, targetValue);
+
+      size++;
+      var dx = attributeValue - attributeMean.Get();
+      var dy = targetValue - targetMean.Get();
+      attributeMean.Add(dx / size);
+      targetMean.Add(dy / size);
+      var dx2 = attributeValue - attributeMean.Get();
+      var dy2 = targetValue - targetMean.Get();
+      attributeVarSum.Add(dx * dx2);
+      targetVarSum.Add(dy * dy2);
+      comoment.Add(dx * dy2);
+
+      ssr.Add(predictOld * Predict(attributeValue, targetValue));
+    }
+
+    public void Remove(double attributeValue, double targetValue) {
+      var predictOld = Predict(attributeValue, targetValue);
+
+      var dx2 = attributeValue - attributeMean.Get();
+      var dy2 = targetValue - targetMean.Get();
+      attributeMean.Mul(size / (size - 1.0));
+      targetMean.Mul(size / (size - 1.0));
+      attributeMean.Add(-attributeValue / (size - 1.0));
+      targetMean.Add(-targetValue / (size - 1.0));
+      var dx = attributeValue - attributeMean.Get();
+      var dy = targetValue - targetMean.Get();
+      attributeVarSum.Add(-dx * dx2);
+      targetVarSum.Add(-dy * dy2);
+      comoment.Add(-dx * dy2);
+      size--;
+
+      ssr.Add(-predictOld * Predict(attributeValue, targetValue));
+    }
+
+    private double Predict(double attributeValue, double targetValue) {
+      return targetValue - Alpha - Beta * attributeValue;
+    }
+  }
+}
+ No newline at end of file