Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/HeuristicLab.Algorithms.DataAnalysis.M5-3.4.csproj
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/HeuristicLab.Algorithms.DataAnalysis.M5-3.4.csproj	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/HeuristicLab.Algorithms.DataAnalysis.M5-3.4.csproj	(revision 17078)
@@ -0,0 +1,213 @@
+﻿<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{CCC5D1F9-0551-4BF3-8898-E4C44B52E5A6}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>HeuristicLab.Algorithms.DataAnalysis.M5</RootNamespace>
+    <AssemblyName>HeuristicLab.Algorithms.DataAnalysis.M5-3.4</AssemblyName>
+    <TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
+    <TargetFrameworkProfile />
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>..\..\bin\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>..\..\bin\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup>
+    <SignAssembly>true</SignAssembly>
+  </PropertyGroup>
+  <PropertyGroup>
+    <AssemblyOriginatorKeyFile>HeuristicLab.snk</AssemblyOriginatorKeyFile>
+  </PropertyGroup>
+  <PropertyGroup>
+    <StartupObject />
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="ALGLIB-3.7.0, Version=3.7.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\bin\ALGLIB-3.7.0.dll</HintPath>
+      <Private>False</Private>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Data.Linq" />
+    <Reference Include="System.Drawing" />
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Data.DataSetExtensions" />
+    <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Net.Http" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="HeuristicLab.snk" />
+    <None Include="Plugin.cs.frame" />
+    <None Include="Properties\AssemblyInfo.cs.frame" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="Interfaces\ILeafModel.cs" />
+    <Compile Include="Interfaces\IM5Model.cs" />
+    <Compile Include="Interfaces\IPruning.cs" />
+    <Compile Include="Interfaces\ISplitter.cs" />
+    <Compile Include="LeafModels\ComponentReducedLinearModel.cs" />
+    <Compile Include="LeafModels\DampenedModel.cs" />
+    <Compile Include="LeafModels\PreconstructedLinearModel.cs" />
+    <Compile Include="LeafTypes\ComplexLeaf.cs" />
+    <Compile Include="LeafTypes\ComponentReductionLinearLeaf.cs" />
+    <Compile Include="LeafTypes\ConstantLeaf.cs" />
+    <Compile Include="LeafTypes\GaussianProcessLeaf.cs" />
+    <Compile Include="LeafTypes\LeafBase.cs" />
+    <Compile Include="LeafTypes\LinearLeaf.cs" />
+    <Compile Include="LeafTypes\M5Leaf.cs" />
+    <Compile Include="LeafTypes\M5regLeaf.cs" />
+    <Compile Include="M5Regression.cs" />
+    <Compile Include="M5Utilities\PrincipleComponentTransformation.cs" />
+    <Compile Include="M5Utilities\RegressionTreeAnalyzer.cs" />
+    <Compile Include="M5Utilities\RegressionTreeParameters.cs" />
+    <Compile Include="M5Utilities\RegressionTreeUtilities.cs" />
+    <Compile Include="MetaModels\RegressionNodeModel.cs" />
+    <Compile Include="MetaModels\RegressionNodeTreeModel.cs" />
+    <Compile Include="MetaModels\RegressionRuleModel.cs" />
+    <Compile Include="MetaModels\RegressionRuleSetModel.cs" />
+    <Compile Include="Plugin.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Pruning\ComplexityPruning.cs" />
+    <Compile Include="Pruning\NoPruning.cs" />
+    <Compile Include="Splitting\CorrelationImpurityCalculator.cs" />
+    <Compile Include="Splitting\CorrelationSplitter.cs" />
+    <Compile Include="Splitting\M5Splitter.cs" />
+    <Compile Include="Splitting\NeumaierSum.cs" />
+    <Compile Include="Splitting\OrderImpurityCalculator.cs" />
+    <Compile Include="Splitting\SplitterBase.cs" />
+    <Compile Include="Splitting\UnivariateOnlineLR.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\HeuristicLab.Algorithms.DataAnalysis.Glmnet\3.4\HeuristicLab.Algorithms.DataAnalysis.Glmnet-3.4.csproj">
+      <Project>{4c7f6d8a-b279-4898-acd6-7be39111def9}</Project>
+      <Name>HeuristicLab.Algorithms.DataAnalysis.Glmnet-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Algorithms.DataAnalysis\3.4\HeuristicLab.Algorithms.DataAnalysis-3.4.csproj">
+      <Project>{2e782078-fa81-4b70-b56f-74ce38dac6c8}</Project>
+      <Name>HeuristicLab.Algorithms.DataAnalysis-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Analysis\3.3\HeuristicLab.Analysis-3.3.csproj">
+      <Project>{887425b4-4348-49ed-a457-b7d2c26ddbf9}</Project>
+      <Name>HeuristicLab.Analysis-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Collections\3.3\HeuristicLab.Collections-3.3.csproj">
+      <Project>{958b43bc-cc5c-4fa2-8628-2b3b01d890b6}</Project>
+      <Name>HeuristicLab.Collections-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Common\3.3\HeuristicLab.Common-3.3.csproj">
+      <Project>{a9ad58b9-3ef9-4cc1-97e5-8d909039ff5c}</Project>
+      <Name>HeuristicLab.Common-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Core\3.3\HeuristicLab.Core-3.3.csproj">
+      <Project>{c36bd924-a541-4a00-afa8-41701378ddc5}</Project>
+      <Name>HeuristicLab.Core-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Data\3.3\HeuristicLab.Data-3.3.csproj">
+      <Project>{bbab9df5-5ef3-4ba8-ade9-b36e82114937}</Project>
+      <Name>HeuristicLab.Data-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Encodings.PermutationEncoding\3.3\HeuristicLab.Encodings.PermutationEncoding-3.3.csproj">
+      <Project>{dbecb8b0-b166-4133-baf1-ed67c3fd7fca}</Project>
+      <Name>HeuristicLab.Encodings.PermutationEncoding-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Encodings.SymbolicExpressionTreeEncoding\3.4\HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4.csproj">
+      <Project>{06D4A186-9319-48A0-BADE-A2058D462EEA}</Project>
+      <Name>HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Optimization\3.3\HeuristicLab.Optimization-3.3.csproj">
+      <Project>{14ab8d24-25bc-400c-a846-4627aa945192}</Project>
+      <Name>HeuristicLab.Optimization-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Parameters\3.3\HeuristicLab.Parameters-3.3.csproj">
+      <Project>{56F9106A-079F-4C61-92F6-86A84C2D84B7}</Project>
+      <Name>HeuristicLab.Parameters-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.PluginInfrastructure\3.3\HeuristicLab.PluginInfrastructure-3.3.csproj">
+      <Project>{94186A6A-5176-4402-AE83-886557B53CCA}</Project>
+      <Name>HeuristicLab.PluginInfrastructure-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.DataAnalysis\3.4\HeuristicLab.Problems.DataAnalysis-3.4.csproj">
+      <Project>{df87c13e-a889-46ff-8153-66dcaa8c5674}</Project>
+      <Name>HeuristicLab.Problems.DataAnalysis-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.Instances\3.3\HeuristicLab.Problems.Instances-3.3.csproj">
+      <Project>{3540e29e-4793-49e7-8ee2-fea7f61c3994}</Project>
+      <Name>HeuristicLab.Problems.Instances-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Random\3.3\HeuristicLab.Random-3.3.csproj">
+      <Project>{f4539fb6-4708-40c9-be64-0a1390aea197}</Project>
+      <Name>HeuristicLab.Random-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+  </ItemGroup>
+  <ItemGroup>
+    <Reference Include="HEAL.Attic, Version=1.0.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\bin\HEAL.Attic.dll</HintPath>
+      <Private>False</Private>
+    </Reference>
+  </ItemGroup>
+  <ItemGroup />
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+  <PropertyGroup>
+    <PreBuildEvent Condition=" '$(OS)' == 'Windows_NT' ">set Path=%25Path%25;$(ProjectDir);$(SolutionDir)
+set ProjectDir=$(ProjectDir)
+set SolutionDir=$(SolutionDir)
+set Outdir=$(Outdir)
+
+call PreBuildEvent.cmd
+</PreBuildEvent>
+    <PreBuildEvent Condition=" '$(OS)' != 'Windows_NT' ">
+export ProjectDir=$(ProjectDir)
+export SolutionDir=$(SolutionDir)
+
+$SolutionDir/PreBuildEvent.sh
+</PreBuildEvent>
+  </PropertyGroup>
+</Project>
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Interfaces/ILeafModel.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Interfaces/ILeafModel.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Interfaces/ILeafModel.cs	(revision 17078)
@@ -0,0 +1,38 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Threading;
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("2A4CB43C-51EB-47AF-AEDD-9B84B27D318B")]
+  public interface ILeafModel : IParameterizedNamedItem {
+    bool ProvidesConfidence { get; }
+    int MinLeafSize(IRegressionProblemData pd);
+
+    void Initialize(IScope states);
+    void Build(RegressionNodeTreeModel tree, IReadOnlyList<int> trainingRows, IScope stateScope, CancellationToken cancellationToken);
+    IRegressionModel BuildModel(IReadOnlyList<int> rows, RegressionTreeParameters parameters, CancellationToken cancellation, out int numParams);
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Interfaces/IM5Model.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Interfaces/IM5Model.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Interfaces/IM5Model.cs	(revision 17078)
@@ -0,0 +1,35 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Threading;
+using HeuristicLab.Core;
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("A5399E6A-6A4D-4616-A1CD-CE12FE670F12")]
+  public interface IM5Model : IRegressionModel {
+    void Build(IReadOnlyList<int> trainingRows, IReadOnlyList<int> pruningRows, IScope stateScope, ResultCollection results, CancellationToken cancellationToken);
+    void Update(IReadOnlyList<int> rows, IScope stateScope, CancellationToken cancellationToken);
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Interfaces/IPruning.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Interfaces/IPruning.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Interfaces/IPruning.cs	(revision 17078)
@@ -0,0 +1,35 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+using System.Collections.Generic;
+using System.Threading;
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("5CAADC16-DCF2-4562-A4DF-C3D5BA1E02A5")]
+  public interface IPruning : IParameterizedNamedItem {
+    int MinLeafSize(IRegressionProblemData pd, ILeafModel leafModel);
+
+    void Initialize(IScope states);
+    void Prune(RegressionNodeTreeModel treeModel, IReadOnlyList<int> trainingRows, IReadOnlyList<int> pruningRows, IScope scope, CancellationToken cancellationToken);
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Interfaces/ISplitter.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Interfaces/ISplitter.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Interfaces/ISplitter.cs	(revision 17078)
@@ -0,0 +1,32 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+using System.Collections.Generic;
+using System.Threading;
+using HeuristicLab.Core;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("A6FB3D68-B298-4C89-9FD9-2D1415D131F5")]
+  public interface ISplitter : IParameterizedNamedItem {
+    void Initialize(IScope states);
+    void Split(RegressionNodeTreeModel tree, IReadOnlyList<int> trainingRows, IScope stateScope, CancellationToken cancellationToken);
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafModels/ComponentReducedLinearModel.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafModels/ComponentReducedLinearModel.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafModels/ComponentReducedLinearModel.cs	(revision 17078)
@@ -0,0 +1,69 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("4E5B8317-648D-4A5A-A3F7-A1A5BEB9AA69")]
+  public sealed class ComponentReducedLinearModel : RegressionModel {
+    [Storable]
+    private IRegressionModel Model;
+    [Storable]
+    private PrincipleComponentTransformation Pca;
+
+    [StorableConstructor]
+    private ComponentReducedLinearModel(StorableConstructorFlag _) : base(_) { }
+    private ComponentReducedLinearModel(ComponentReducedLinearModel original, Cloner cloner) : base(original, cloner) {
+      Model = cloner.Clone(original.Model);
+      Pca = cloner.Clone(original.Pca);
+    }
+
+    public ComponentReducedLinearModel(string targetVariable, IRegressionModel model, PrincipleComponentTransformation pca) : base(targetVariable) {
+      Model = model;
+      Pca = pca;
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new ComponentReducedLinearModel(this, cloner);
+    }
+
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return Model.VariablesUsedForPrediction; }
+    }
+
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      var data = ReduceDataset(dataset, rows.ToArray());
+      return Model.GetEstimatedValues(Pca.TransformDataset(data), Enumerable.Range(0, data.Rows));
+    }
+
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RegressionSolution(this, problemData);
+    }
+
+    private IDataset ReduceDataset(IDataset data, IReadOnlyList<int> rows) {
+      return new Dataset(data.DoubleVariables, data.DoubleVariables.Select(v => data.GetDoubleValues(v, rows).ToList()));
+    }
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafModels/DampenedModel.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafModels/DampenedModel.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafModels/DampenedModel.cs	(revision 17078)
@@ -0,0 +1,124 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  //mulitdimensional extension of http://www2.stat.duke.edu/~tjl13/s101/slides/unit6lec3H.pdf
+  [StorableType("42E9766F-207F-47B1-890C-D5DFCF469838")]
+  public class DampenedModel : RegressionModel {
+    [Storable]
+    protected IRegressionModel Model;
+    [Storable]
+    private double Min;
+    [Storable]
+    private double Max;
+    [Storable]
+    private double Dampening;
+
+    [StorableConstructor]
+    protected DampenedModel(StorableConstructorFlag _) : base(_) { }
+    protected DampenedModel(DampenedModel original, Cloner cloner) : base(original, cloner) {
+      Model = cloner.Clone(original.Model);
+      Min = original.Min;
+      Max = original.Max;
+      Dampening = original.Dampening;
+    }
+    protected DampenedModel(IRegressionModel model, IRegressionProblemData pd, double dampening) : base(model.TargetVariable) {
+      Model = model;
+      Min = pd.TargetVariableTrainingValues.Min();
+      Max = pd.TargetVariableTrainingValues.Max();
+      Dampening = dampening;
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new DampenedModel(this, cloner);
+    }
+
+    public static IConfidenceRegressionModel DampenModel(IConfidenceRegressionModel model, IRegressionProblemData pd, double dampening) {
+      return new ConfidenceDampenedModel(model, pd, dampening);
+    }
+    public static IRegressionModel DampenModel(IRegressionModel model, IRegressionProblemData pd, double dampening) {
+      var cmodel = model as IConfidenceRegressionModel;
+      return cmodel != null ? new ConfidenceDampenedModel(cmodel, pd, dampening) : new DampenedModel(model, pd, dampening);
+    }
+
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return Model.VariablesUsedForPrediction; }
+    }
+
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      var slow = Sigmoid(-Dampening);
+      var shigh = Sigmoid(Dampening);
+      foreach (var x in Model.GetEstimatedValues(dataset, rows)) {
+        var y = Rescale(x, Min, Max, -Dampening, Dampening);
+        y = Sigmoid(y);
+        y = Rescale(y, slow, shigh, Min, Max);
+        yield return y;
+      }
+    }
+
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RegressionSolution(this, problemData);
+    }
+
+    private static double Rescale(double x, double oMin, double oMax, double nMin, double nMax) {
+      var d = oMax - oMin;
+      var nd = nMax - nMin;
+      if (d.IsAlmost(0)) {
+        d = 1;
+        nMin += nd / 2;
+        nd = 0;
+      }
+      return ((x - oMin) / d) * nd + nMin;
+    }
+
+    private static double Sigmoid(double x) {
+      return 1 / (1 + Math.Exp(-x));
+    }
+
+
+    [StorableType("CCC93BEC-8796-4D8E-AC58-DD175073A79B")]
+    private sealed class ConfidenceDampenedModel : DampenedModel, IConfidenceRegressionModel {
+      #region HLConstructors
+      [StorableConstructor]
+      private ConfidenceDampenedModel(StorableConstructorFlag _) : base(_) { }
+      private ConfidenceDampenedModel(ConfidenceDampenedModel original, Cloner cloner) : base(original, cloner) { }
+      public ConfidenceDampenedModel(IConfidenceRegressionModel model, IRegressionProblemData pd, double dampening) : base(model, pd, dampening) { }
+      public override IDeepCloneable Clone(Cloner cloner) {
+        return new ConfidenceDampenedModel(this, cloner);
+      }
+      #endregion
+
+      public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
+        return ((IConfidenceRegressionModel)Model).GetEstimatedVariances(dataset, rows);
+      }
+
+      public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+        return new ConfidenceRegressionSolution(this, problemData);
+      }
+    }
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafModels/PreconstructedLinearModel.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafModels/PreconstructedLinearModel.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafModels/PreconstructedLinearModel.cs	(revision 17078)
@@ -0,0 +1,153 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  //mulitdimensional extension of http://www2.stat.duke.edu/~tjl13/s101/slides/unit6lec3H.pdf
+  [StorableType("15F2295C-28C1-48C3-8DCB-9470823C6734")]
+  internal sealed class PreconstructedLinearModel : RegressionModel {
+    [Storable]
+    public Dictionary<string, double> Coefficients { get; private set; }
+    [Storable]
+    public double Intercept { get; private set; }
+
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return Coefficients.Keys; }
+    }
+
+    #region HLConstructors
+    [StorableConstructor]
+    private PreconstructedLinearModel(StorableConstructorFlag _) : base(_) { }
+    private PreconstructedLinearModel(PreconstructedLinearModel original, Cloner cloner) : base(original, cloner) {
+      if (original.Coefficients != null) Coefficients = original.Coefficients.ToDictionary(x => x.Key, x => x.Value);
+      Intercept = original.Intercept;
+    }
+    public PreconstructedLinearModel(Dictionary<string, double> coefficients, double intercept, string targetvariable) : base(targetvariable) {
+      Coefficients = new Dictionary<string, double>(coefficients);
+      Intercept = intercept;
+    }
+    public PreconstructedLinearModel(double intercept, string targetvariable) : base(targetvariable) {
+      Coefficients = new Dictionary<string, double>();
+      Intercept = intercept;
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new PreconstructedLinearModel(this, cloner);
+    }
+    #endregion
+
+    public static PreconstructedLinearModel CreateLinearModel(IRegressionProblemData pd, out double rmse) {
+      return AlternativeCalculation(pd, out rmse);
+    }
+
+    private static PreconstructedLinearModel ClassicCalculation(IRegressionProblemData pd) {
+      var inputMatrix = pd.Dataset.ToArray(pd.AllowedInputVariables.Concat(new[] {
+        pd.TargetVariable
+      }), pd.AllIndices);
+
+      var nFeatures = inputMatrix.GetLength(1) - 1;
+      double[] coefficients;
+
+      alglib.linearmodel lm;
+      alglib.lrreport ar;
+      int retVal;
+      alglib.lrbuild(inputMatrix, inputMatrix.GetLength(0), nFeatures, out retVal, out lm, out ar);
+      if (retVal != 1) throw new ArgumentException("Error in calculation of linear regression solution");
+
+      alglib.lrunpack(lm, out coefficients, out nFeatures);
+      var coeffs = pd.AllowedInputVariables.Zip(coefficients, (s, d) => new {s, d}).ToDictionary(x => x.s, x => x.d);
+      var res = new PreconstructedLinearModel(coeffs, coefficients[nFeatures], pd.TargetVariable);
+      return res;
+    }
+
+    private static PreconstructedLinearModel AlternativeCalculation(IRegressionProblemData pd, out double rmse) {
+      var variables = pd.AllowedInputVariables.ToList();
+      var n = variables.Count;
+      var m = pd.TrainingIndices.Count();
+
+      //Set up X^T
+      var inTr = new double[n + 1, m];
+      for (var i = 0; i < n; i++) {
+        var vdata = pd.Dataset.GetDoubleValues(variables[i], pd.TrainingIndices).ToArray();
+        for (var j = 0; j < m; j++) inTr[i, j] = vdata[j];
+      }
+      for (var i = 0; i < m; i++) inTr[n, i] = 1;
+
+      //Set up y
+      var y = new double[m, 1];
+      var ydata = pd.TargetVariableTrainingValues.ToArray();
+      for (var i = 0; i < m; i++) y[i, 0] = ydata[i];
+
+      //Perform linear regression
+      var aTy = new double[n + 1, 1];
+      var aTa = new double[n + 1, n + 1];
+      var aTyVector = new double[n + 1];
+      int info;
+      alglib.densesolverreport report;
+      double[] coefficients;
+
+      //Perform linear regression
+      alglib.rmatrixgemm(n + 1, 1, m, 1, inTr, 0, 0, 0, y, 0, 0, 0, 0, ref aTy, 0, 0); //aTy = inTr * y;
+      alglib.rmatrixgemm(n + 1, n + 1, m, 1, inTr, 0, 0, 0, inTr, 0, 0, 1, 0, ref aTa, 0, 0); //aTa = inTr * t(inTr) +aTa //
+      alglib.spdmatrixcholesky(ref aTa, n + 1, true);
+      for (var i = 0; i < n + 1; i++) aTyVector[i] = aTy[i, 0];
+      alglib.spdmatrixcholeskysolve(aTa, n + 1, true, aTyVector, out info, out report, out coefficients);
+
+      //if Cholesky calculation fails fall back to classic linear regresseion 
+      if (info != 1) {
+        alglib.linearmodel lm;
+        alglib.lrreport ar;
+        int retVal;
+        var inputMatrix = pd.Dataset.ToArray(pd.AllowedInputVariables.Concat(new[] {
+          pd.TargetVariable
+        }), pd.AllIndices);
+        alglib.lrbuild(inputMatrix, inputMatrix.GetLength(0), n, out retVal, out lm, out ar);
+        if (retVal != 1) throw new ArgumentException("Error in calculation of linear regression solution");
+        alglib.lrunpack(lm, out coefficients, out n);
+      }
+
+      var coeffs = Enumerable.Range(0, n).ToDictionary(i => variables[i], i => coefficients[i]);
+      var model = new PreconstructedLinearModel(coeffs, coefficients[n], pd.TargetVariable);
+      rmse = pd.TrainingIndices.Select(i => pd.Dataset.GetDoubleValue(pd.TargetVariable, i) - model.GetEstimatedValue(pd.Dataset, i)).Sum(r => r * r) / m;
+      rmse = Math.Sqrt(rmse);
+      return model;
+    }
+
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      return rows.Select(row => GetEstimatedValue(dataset, row));
+    }
+
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RegressionSolution(this, problemData);
+    }
+
+    #region helpers
+    private double GetEstimatedValue(IDataset dataset, int row) {
+      return Intercept + (Coefficients.Count == 0 ? 0 : Coefficients.Sum(s => s.Value * dataset.GetDoubleValue(s.Key, row)));
+    }
+    #endregion
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/ComplexLeaf.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/ComplexLeaf.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/ComplexLeaf.cs	(revision 17078)
@@ -0,0 +1,75 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Parameters;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("F34A0ED9-2CF6-4DEE-850D-08790663B66D")]
+  [Item("ComplexLeaf", "A leaf type that uses an arbitrary RegressionAlgorithm to create leaf models")]
+  public sealed class ComplexLeaf : LeafBase {
+    public const string RegressionParameterName = "Regression";
+    public IValueParameter<IDataAnalysisAlgorithm<IRegressionProblem>> RegressionParameter {
+      get { return (IValueParameter<IDataAnalysisAlgorithm<IRegressionProblem>>)Parameters[RegressionParameterName]; }
+    }
+    public IDataAnalysisAlgorithm<IRegressionProblem> Regression {
+      get { return RegressionParameter.Value; }
+      set { RegressionParameter.Value = value; }
+    }
+
+    #region Constructors & Cloning
+    [StorableConstructor]
+    private ComplexLeaf(StorableConstructorFlag _) : base(_) { }
+    private ComplexLeaf(ComplexLeaf original, Cloner cloner) : base(original, cloner) { }
+    public ComplexLeaf() {
+      var regression = new KernelRidgeRegression();
+      Parameters.Add(new ValueParameter<IDataAnalysisAlgorithm<IRegressionProblem>>(RegressionParameterName, "The algorithm creating RegressionModels", regression));
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new ComplexLeaf(this, cloner);
+    }
+    #endregion
+
+    #region IModelType
+    public override bool ProvidesConfidence {
+      get { return false; }
+    }
+    public override IRegressionModel Build(IRegressionProblemData pd, IRandom random, CancellationToken cancellationToken, out int noParameters) {
+      if (pd.Dataset.Rows < MinLeafSize(pd)) throw new ArgumentException("The number of training instances is too small to create a linear model");
+      noParameters = pd.Dataset.Rows + 1;
+      Regression.Problem = new RegressionProblem { ProblemData = pd };
+      var res = RegressionTreeUtilities.RunSubAlgorithm(Regression, random.Next(), cancellationToken);
+      var t = res.Select(x => x.Value).OfType<IRegressionSolution>().FirstOrDefault();
+      if (t == null) throw new ArgumentException("No RegressionSolution was provided by the algorithm");
+      return t.Model;
+    }
+    public override int MinLeafSize(IRegressionProblemData pd) {
+      return 3;
+    }
+    #endregion
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/ComponentReductionLinearLeaf.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/ComponentReductionLinearLeaf.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/ComponentReductionLinearLeaf.cs	(revision 17078)
@@ -0,0 +1,90 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("5730B54C-7A8B-4CA7-8F37-7FF3F9848CD2")]
+  [Item("ComponentReductionLinearLeaf", "A leaf type that uses principle component analysis to create smaller linear models as leaf models")]
+  public class ComponentReductionLinearLeaf : LeafBase {
+    public const string NumberOfComponentsParameterName = "NoComponents";
+    public IFixedValueParameter<IntValue> NumberOfCompontentsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NumberOfComponentsParameterName]; }
+    }
+    public int NumberOfComponents {
+      get { return NumberOfCompontentsParameter.Value.Value; }
+      set { NumberOfCompontentsParameter.Value.Value = value; }
+    }
+
+    #region Constructors & Cloning
+    [StorableConstructor]
+    protected ComponentReductionLinearLeaf(StorableConstructorFlag _) : base(_) { }
+    protected ComponentReductionLinearLeaf(ComponentReductionLinearLeaf original, Cloner cloner) : base(original, cloner) { }
+    public ComponentReductionLinearLeaf() {
+      Parameters.Add(new FixedValueParameter<IntValue>(NumberOfComponentsParameterName, "The maximum number of principle components used (default=10)", new IntValue(10)));
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new ComponentReductionLinearLeaf(this, cloner);
+    }
+    #endregion
+
+    #region IModelType
+    public override bool ProvidesConfidence {
+      get { return false; }
+    }
+
+    public override IRegressionModel Build(IRegressionProblemData pd, IRandom random,
+      CancellationToken cancellationToken, out int numberOfParameters) {
+      var pca = PrincipleComponentTransformation.CreateProjection(pd.Dataset, pd.TrainingIndices, pd.AllowedInputVariables, normalize: true);
+      var pcdata = pca.TransformProblemData(pd);
+      ComponentReducedLinearModel bestModel = null;
+      var bestCvrmse = double.MaxValue;
+      numberOfParameters = 1;
+      for (var i = 1; i <= Math.Min(NumberOfComponents, pd.AllowedInputVariables.Count()); i++) {
+        var pd2 = (IRegressionProblemData)pcdata.Clone();
+        var inputs = new HashSet<string>(pca.ComponentNames.Take(i));
+        foreach (var v in pd2.InputVariables.CheckedItems.ToArray())
+          pd2.InputVariables.SetItemCheckedState(v.Value, inputs.Contains(v.Value.Value));
+        double rmse;
+        var model = PreconstructedLinearModel.CreateLinearModel(pd2, out rmse);
+        if (rmse > bestCvrmse) continue;
+        bestModel = new ComponentReducedLinearModel(pd2.TargetVariable, model, pca);
+        numberOfParameters = i + 1;
+        bestCvrmse = rmse;
+      }
+      return bestModel;
+    }
+
+    public override int MinLeafSize(IRegressionProblemData pd) {
+      return NumberOfComponents + 2;
+    }
+    #endregion
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/ConstantLeaf.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/ConstantLeaf.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/ConstantLeaf.cs	(revision 17078)
@@ -0,0 +1,59 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("F3E94907-C5FF-4658-A870-8013C61DD2E1")]
+  [Item("ConstantLeaf", "A leaf type that uses constant models as leaf models")]
+  public class ConstantLeaf : LeafBase {
+    #region Constructors & Cloning
+    [StorableConstructor]
+    protected ConstantLeaf(StorableConstructorFlag _) : base(_) { }
+    protected ConstantLeaf(ConstantLeaf original, Cloner cloner) : base(original, cloner) { }
+    public ConstantLeaf() { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new ConstantLeaf(this, cloner);
+    }
+    #endregion
+
+    #region IModelType
+    public override bool ProvidesConfidence {
+      get { return false; }
+    }
+    public override IRegressionModel Build(IRegressionProblemData pd, IRandom random, CancellationToken cancellationToken, out int numberOfParameters) {
+      if (pd.Dataset.Rows < MinLeafSize(pd)) throw new ArgumentException("The number of training instances is too small to create a linear model");
+      numberOfParameters = 1;
+      return new PreconstructedLinearModel(pd.Dataset.GetDoubleValues(pd.TargetVariable).Average(), pd.TargetVariable);
+    }
+
+    public override int MinLeafSize(IRegressionProblemData pd) {
+      return 0;
+    }
+    #endregion
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/GaussianProcessLeaf.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/GaussianProcessLeaf.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/GaussianProcessLeaf.cs	(revision 17078)
@@ -0,0 +1,111 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("852B9F7D-9C2B-4574-BB71-EE70106EA809")]
+  [Item("GaussianProcessLeaf", "A leaf type that uses Gaussian process models as leaf models.")]
+  public class GaussianProcessLeaf : LeafBase {
+    #region ParameterNames
+    public const string TriesParameterName = "Tries";
+    public const string RegressionParameterName = "Regression";
+    #endregion
+
+    #region ParameterProperties
+    public IFixedValueParameter<IntValue> TriesParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[TriesParameterName]; }
+    }
+    public IValueParameter<GaussianProcessRegression> RegressionParameter {
+      get { return (IValueParameter<GaussianProcessRegression>)Parameters[RegressionParameterName]; }
+    }
+    #endregion
+
+    #region Properties
+    public int Tries {
+      get { return TriesParameter.Value.Value; }
+      set { TriesParameter.Value.Value = value; }
+    }
+    public GaussianProcessRegression Regression {
+      get { return RegressionParameter.Value; }
+      set { RegressionParameter.Value = value; }
+    }
+    #endregion
+
+    #region Constructors & Cloning
+    [StorableConstructor]
+    protected GaussianProcessLeaf(StorableConstructorFlag _) : base(_) { }
+    protected GaussianProcessLeaf(GaussianProcessLeaf original, Cloner cloner) : base(original, cloner) { }
+    public GaussianProcessLeaf() {
+      var gp = new GaussianProcessRegression();
+      gp.CovarianceFunctionParameter.Value = new CovarianceRationalQuadraticIso();
+      gp.MeanFunctionParameter.Value = new MeanLinear();
+
+      Parameters.Add(new FixedValueParameter<IntValue>(TriesParameterName, "Number of restarts (default = 10)", new IntValue(10)));
+      Parameters.Add(new ValueParameter<GaussianProcessRegression>(RegressionParameterName, "The algorithm creating Gaussian process models", gp));
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessLeaf(this, cloner);
+    }
+    #endregion
+
+    #region IModelType
+    public override bool ProvidesConfidence {
+      get { return true; }
+    }
+
+    public override IRegressionModel Build(IRegressionProblemData pd, IRandom random, CancellationToken cancellationToken, out int numberOfParameters) {
+      if (pd.Dataset.Rows < MinLeafSize(pd)) throw new ArgumentException("The number of training instances is too small to create a Gaussian process model");
+      Regression.Problem = new RegressionProblem { ProblemData = pd };
+      var cvscore = double.MaxValue;
+      GaussianProcessRegressionSolution sol = null;
+
+      for (var i = 0; i < Tries; i++) {
+        var res = RegressionTreeUtilities.RunSubAlgorithm(Regression, random.Next(), cancellationToken);
+        var t = res.Select(x => x.Value).OfType<GaussianProcessRegressionSolution>().FirstOrDefault();
+        var score = ((DoubleValue)res["Negative log pseudo-likelihood (LOO-CV)"].Value).Value;
+        if (score >= cvscore || t == null || double.IsNaN(t.TrainingRSquared)) continue;
+        cvscore = score;
+        sol = t;
+      }
+      Regression.Runs.Clear();
+      if (sol == null) throw new ArgumentException("Could not create Gaussian process model");
+
+      numberOfParameters = pd.Dataset.Rows + 1
+                           + Regression.CovarianceFunction.GetNumberOfParameters(pd.AllowedInputVariables.Count())
+                           + Regression.MeanFunction.GetNumberOfParameters(pd.AllowedInputVariables.Count());
+      return sol.Model;
+    }
+
+    public override int MinLeafSize(IRegressionProblemData pd) {
+      return 3;
+    }
+    #endregion
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/LeafBase.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/LeafBase.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/LeafBase.cs	(revision 17078)
@@ -0,0 +1,167 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("F3A9CCD4-975F-4F55-BE24-3A3E932591F6")]
+  public abstract class LeafBase : ParameterizedNamedItem, ILeafModel {
+    public const string LeafBuildingStateVariableName = "LeafBuildingState";
+    public const string UseDampeningParameterName = "UseDampening";
+    public const string DampeningParameterName = "DampeningStrength";
+
+    public IFixedValueParameter<DoubleValue> DampeningParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[DampeningParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> UseDampeningParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[UseDampeningParameterName]; }
+    }
+
+    public bool UseDampening {
+      get { return UseDampeningParameter.Value.Value; }
+      set { UseDampeningParameter.Value.Value = value; }
+    }
+    public double Dampening {
+      get { return DampeningParameter.Value.Value; }
+      set { DampeningParameter.Value.Value = value; }
+    }
+
+    #region Constructors & Cloning
+    [StorableConstructor]
+    protected LeafBase(StorableConstructorFlag _) : base(_) { }
+    protected LeafBase(LeafBase original, Cloner cloner) : base(original, cloner) { }
+    protected LeafBase() {
+      Parameters.Add(new FixedValueParameter<BoolValue>(UseDampeningParameterName, "Whether logistic dampening should be used to prevent extreme extrapolation (default=false)", new BoolValue(false)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(DampeningParameterName, "Determines the strength of logistic dampening. Must be > 0.0. Larger numbers lead to more conservative predictions. (default=1.5)", new DoubleValue(1.5)));
+    }
+    #endregion
+
+    #region IModelType
+    public abstract bool ProvidesConfidence { get; }
+    public abstract int MinLeafSize(IRegressionProblemData pd);
+
+    public void Initialize(IScope states) {
+      states.Variables.Add(new Variable(LeafBuildingStateVariableName, new LeafBuildingState()));
+    }
+
+    public void Build(RegressionNodeTreeModel tree, IReadOnlyList<int> trainingRows, IScope stateScope, CancellationToken cancellationToken) {
+      var parameters = (RegressionTreeParameters)stateScope.Variables[M5Regression.RegressionTreeParameterVariableName].Value;
+      var state = (LeafBuildingState)stateScope.Variables[LeafBuildingStateVariableName].Value;
+
+      if (state.Code == 0) {
+        state.FillLeafs(tree, trainingRows, parameters.Data);
+        state.Code = 1;
+      }
+      while (state.nodeQueue.Count != 0) {
+        var n = state.nodeQueue.Peek();
+        var t = state.trainingRowsQueue.Peek();
+        int numP;
+        n.SetLeafModel(BuildModel(t, parameters, cancellationToken, out numP));
+        state.nodeQueue.Dequeue();
+        state.trainingRowsQueue.Dequeue();
+      }
+    }
+
+    public IRegressionModel BuildModel(IReadOnlyList<int> rows, RegressionTreeParameters parameters, CancellationToken cancellation, out int numberOfParameters) {
+      var reducedData = RegressionTreeUtilities.ReduceDataset(parameters.Data, rows, parameters.AllowedInputVariables.ToArray(), parameters.TargetVariable);
+      var pd = new RegressionProblemData(reducedData, parameters.AllowedInputVariables.ToArray(), parameters.TargetVariable);
+      pd.TrainingPartition.Start = 0;
+      pd.TrainingPartition.End = pd.TestPartition.Start = pd.TestPartition.End = reducedData.Rows;
+
+      int numP;
+      var model = Build(pd, parameters.Random, cancellation, out numP);
+      if (UseDampening && Dampening > 0.0) {
+        model = DampenedModel.DampenModel(model, pd, Dampening);
+      }
+
+      numberOfParameters = numP;
+      cancellation.ThrowIfCancellationRequested();
+      return model;
+    }
+
+    public abstract IRegressionModel Build(IRegressionProblemData pd, IRandom random, CancellationToken cancellationToken, out int numberOfParameters);
+    #endregion
+
+    [StorableType("495243C0-6C15-4328-B30D-FFBFA0F54DCB")]
+    public class LeafBuildingState : Item {
+      [Storable]
+      public Queue<RegressionNodeModel> nodeQueue = new Queue<RegressionNodeModel>();
+      [Storable]
+      public Queue<IReadOnlyList<int>> trainingRowsQueue = new Queue<IReadOnlyList<int>>();
+
+      //State.Code values denote the current action (for pausing)
+      //0...nothing has been done;
+      //1...building models;
+      [Storable]
+      public int Code = 0;
+
+      #region HLConstructors & Cloning
+      [StorableConstructor]
+      protected LeafBuildingState(StorableConstructorFlag _) : base(_) { }
+      protected LeafBuildingState(LeafBuildingState original, Cloner cloner) : base(original, cloner) {
+        nodeQueue = new Queue<RegressionNodeModel>(original.nodeQueue.Select(cloner.Clone));
+        trainingRowsQueue = new Queue<IReadOnlyList<int>>(original.trainingRowsQueue.Select(x => (IReadOnlyList<int>)x.ToArray()));
+        Code = original.Code;
+      }
+      public LeafBuildingState() { }
+      public override IDeepCloneable Clone(Cloner cloner) {
+        return new LeafBuildingState(this, cloner);
+      }
+      #endregion
+
+      public void FillLeafs(RegressionNodeTreeModel tree, IReadOnlyList<int> trainingRows, IDataset data) {
+        var helperQueue = new Queue<RegressionNodeModel>();
+        var trainingHelperQueue = new Queue<IReadOnlyList<int>>();
+        nodeQueue.Clear();
+        trainingRowsQueue.Clear();
+
+        helperQueue.Enqueue(tree.Root);
+        trainingHelperQueue.Enqueue(trainingRows);
+
+        while (helperQueue.Count != 0) {
+          var n = helperQueue.Dequeue();
+          var t = trainingHelperQueue.Dequeue();
+          if (n.IsLeaf) {
+            nodeQueue.Enqueue(n);
+            trainingRowsQueue.Enqueue(t);
+            continue;
+          }
+
+          IReadOnlyList<int> leftTraining, rightTraining;
+          RegressionTreeUtilities.SplitRows(t, data, n.SplitAttribute, n.SplitValue, out leftTraining, out rightTraining);
+
+          helperQueue.Enqueue(n.Left);
+          helperQueue.Enqueue(n.Right);
+          trainingHelperQueue.Enqueue(leftTraining);
+          trainingHelperQueue.Enqueue(rightTraining);
+        }
+      }
+    }
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/LinearLeaf.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/LinearLeaf.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/LinearLeaf.cs	(revision 17078)
@@ -0,0 +1,61 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("347CA25D-FB37-4C4F-9B61-9D79288B2B28")]
+  [Item("LinearLeaf", "A leaf type that uses linear models as leaf models. This is the standard for M5' regression")]
+  public class LinearLeaf : LeafBase {
+    #region Constructors & Cloning
+    [StorableConstructor]
+    protected LinearLeaf(StorableConstructorFlag _) : base(_) { }
+    protected LinearLeaf(LinearLeaf original, Cloner cloner) : base(original, cloner) { }
+    public LinearLeaf() { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new LinearLeaf(this, cloner);
+    }
+    #endregion
+
+    #region IModelType
+    public override bool ProvidesConfidence {
+      get { return true; }
+    }
+    public override IRegressionModel Build(IRegressionProblemData pd, IRandom random, CancellationToken cancellationToken, out int numberOfParameters) {
+      if (pd.Dataset.Rows < MinLeafSize(pd)) throw new ArgumentException("The number of training instances is too small to create a linear model");
+      double rmse, cvRmse;
+      numberOfParameters = pd.AllowedInputVariables.Count() + 1;
+      var res = LinearRegression.CreateSolution(pd, out rmse, out cvRmse);
+      return res.Model;
+    }
+
+    public override int MinLeafSize(IRegressionProblemData pd) {
+      return pd.AllowedInputVariables.Count() == 1 ? 2 : pd.AllowedInputVariables.Count() + 2;
+    }
+    #endregion
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/M5Leaf.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/M5Leaf.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/M5Leaf.cs	(revision 17078)
@@ -0,0 +1,198 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("58517042-5318-4087-B098-AC75F0208BA0")]
+  [Item("M5Leaf", "A leaf type that uses regularized linear models with feature selection as leaf models.")]
+  public sealed class M5Leaf : LeafBase {
+    #region Constructors & Cloning
+    [StorableConstructor]
+    private M5Leaf(StorableConstructorFlag _) : base(_) { }
+    private M5Leaf(M5Leaf original, Cloner cloner) : base(original, cloner) { }
+    public M5Leaf() { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new M5Leaf(this, cloner);
+    }
+    #endregion
+
+    #region IModelType
+    public override bool ProvidesConfidence {
+      get { return false; }
+    }
+    public override IRegressionModel Build(IRegressionProblemData pd, IRandom random, CancellationToken cancellationToken, out int numberOfParameters) {
+      if (pd.Dataset.Rows == 0) throw new ArgumentException("The number of training instances is too small to create an M5 leaf model");
+
+      if (pd.Dataset.Rows == 1)
+        return new ConstantLeaf().Build(pd, random, cancellationToken, out numberOfParameters);
+
+      var means = pd.AllowedInputVariables.ToDictionary(n => n, n => pd.Dataset.GetDoubleValues(n, pd.TrainingIndices).Average());
+      var variances = pd.AllowedInputVariables.ToDictionary(n => n, n => pd.Dataset.GetDoubleValues(n, pd.TrainingIndices).Variance());
+      var used = pd.AllowedInputVariables.Where(v => !variances[v].IsAlmost(0.0)).ToList();
+
+      var targetMean = pd.TargetVariableTrainingValues.Average();
+      var targetVariance = pd.TargetVariableTrainingValues.Variance();
+
+      var model = FindBestModel(variances, means, targetMean, targetVariance, pd, used);
+      numberOfParameters = 1 + model.Coefficients.Count;
+      return model;
+    }
+    public override int MinLeafSize(IRegressionProblemData pd) {
+      return 1;
+    }
+    #endregion
+
+    private static PreconstructedLinearModel FindBestModel(Dictionary<string, double> variances, Dictionary<string, double> means, double yMean, double yVariance, IRegressionProblemData pd, IList<string> variables) {
+      Dictionary<string, double> coeffs;
+      double intercept;
+      do {
+        coeffs = DoRegression(pd, variables, variances, means, yMean, 1.0e-8, out intercept);
+        variables = DeselectColinear(variances, coeffs, yVariance, pd, variables);
+      }
+      while (coeffs.Count != variables.Count);
+      var numAtts = variables.Count;
+      var numInst = pd.TrainingIndices.Count();
+      var fullMse = CalculateSE(coeffs, intercept, pd, variables);
+      var akaike = 1.0 * (numInst - numAtts) + 2 * numAtts;
+
+      var improved = true;
+      var currentNumAttributes = numAtts;
+
+      while (improved && currentNumAttributes > 1) {
+        improved = false;
+        currentNumAttributes--;
+        // Find attribute with smallest SC (variance-scaled coefficient)
+        var candidate = variables.ToDictionary(v => v, v => Math.Abs(coeffs[v] * Math.Sqrt(variances[v] / yVariance)))
+          .OrderBy(x => x.Value).Select(x => x.Key).First();
+
+        var currVariables = variables.Where(v => !v.Equals(candidate)).ToList();
+        var currentIntercept = 0.0;
+        var currentCoeffs = DoRegression(pd, currVariables, variances, means, yMean, 1.0e-8, out currentIntercept);
+        var currentMse = CalculateSE(currentCoeffs, currentIntercept, pd, currVariables);
+        var currentAkaike = currentMse / fullMse * (numInst - numAtts) + 2 * currentNumAttributes;
+
+        if (!(currentAkaike < akaike)) continue;
+        improved = true;
+        akaike = currentAkaike;
+        coeffs = currentCoeffs;
+        intercept = currentIntercept;
+        variables = currVariables;
+      }
+
+      var pd2 = new RegressionProblemData(pd.Dataset, variables, pd.TargetVariable);
+      pd2.TestPartition.End = pd.TestPartition.End;
+      pd2.TestPartition.Start = pd.TestPartition.Start;
+      pd2.TrainingPartition.End = pd.TrainingPartition.End;
+      pd2.TrainingPartition.Start = pd.TrainingPartition.Start;
+
+      return new PreconstructedLinearModel(coeffs, intercept, pd.TargetVariable);
+    }
+
+    private static Dictionary<string, double> DoRegression(IRegressionProblemData pd, IList<string> variables, Dictionary<string, double> variances, Dictionary<string, double> means, double yMean, double ridge, out double intercept) {
+
+      var n = variables.Count;
+      var m = pd.TrainingIndices.Count();
+
+      var inTr = new double[n, m];
+      for (var i = 0; i < n; i++) {
+        var v = variables[i];
+        var vdata = pd.Dataset.GetDoubleValues(v, pd.TrainingIndices).ToArray();
+        var sd = Math.Sqrt(variances[v]);
+        var mean = means[v];
+        for (var j = 0; j < m; j++) {
+          inTr[i, j] = (vdata[j] - mean) / sd;
+        }
+      }
+
+      var y = new double[m, 1];
+      var ydata = pd.TargetVariableTrainingValues.ToArray();
+      for (var i = 0; i < m; i++)
+        y[i, 0] = ydata[i]; //no scaling for targets;
+
+
+      var aTy = new double[n, 1];
+      alglib.rmatrixgemm(n, 1, m, 1, inTr, 0, 0, 0, y, 0, 0, 0, 0, ref aTy, 0, 0); //aTy = inTr * y;
+      var aTa = new double[n, n];
+      alglib.rmatrixgemm(n, n, m, 1, inTr, 0, 0, 0, inTr, 0, 0, 1, 0, ref aTa, 0, 0); //aTa = inTr * t(inTr) +aTa //
+
+      var aTaDecomp = new double[n, n];
+      bool success;
+      var tries = 0;
+      double[] coefficients = null;
+      do {
+        for (var i = 0; i < n; i++) aTa[i, i] += ridge; // add ridge to diagonal to enforce singularity
+        try {
+          //solve "aTa * coefficients = aTy" for coefficients;
+          Array.Copy(aTa, 0, aTaDecomp, 0, aTa.Length);
+          alglib.spdmatrixcholesky(ref aTaDecomp, n, true);
+          int info;
+          alglib.densesolverreport report;
+          alglib.spdmatrixcholeskysolve(aTaDecomp, n, true, ydata, out info, out report, out coefficients);
+
+          if (info != 1) throw new Exception();
+          success = true;
+        }
+        catch (Exception) {
+          for (var i = 0; i < n; i++) aTa[i, i] -= ridge;
+          ridge *= 10; // increase ridge;
+          success = false;
+        }
+        finally {
+          tries++;
+        }
+      }
+      while (!success && tries < 100);
+      if (coefficients == null || coefficients.Length != n) throw new ArgumentException("No linear model could be built");
+
+      intercept = yMean;
+      var res = new Dictionary<string, double>();
+      for (var i = 0; i < n; i++) {
+        var v = variables[i];
+        res.Add(v, coefficients[i] /= Math.Sqrt(variances[v]));
+        intercept -= coefficients[i] * means[v];
+      }
+
+      return res;
+    }
+
+    private static IList<string> DeselectColinear(Dictionary<string, double> variances, Dictionary<string, double> coeffs, double yVariance, IRegressionProblemData pd, IList<string> variables) {
+      var candidates = variables.ToDictionary(v => v, v => Math.Abs(coeffs[v] * Math.Sqrt(variances[v] / yVariance))).Where(x => x.Value > 1.5).OrderBy(x => -x.Value).ToList();
+      if (candidates.Count == 0) return variables;
+      var c = candidates.First().Key;
+      return variables.Where(v => !v.Equals(c)).ToList();
+    }
+
+    private static double CalculateSE(Dictionary<string, double> coefficients, double intercept, IRegressionProblemData pd, IList<string> variables) {
+      return pd.TrainingIndices.Select(i => RegressionPrediction(i, pd, variables, coefficients, intercept) - pd.Dataset.GetDoubleValue(pd.TargetVariable, i)).Select(error => error * error).Sum();
+    }
+    private static double RegressionPrediction(int i, IRegressionProblemData pd, IList<string> variables, Dictionary<string, double> coefficients, double intercept) {
+      return intercept + variables.Sum(v => pd.Dataset.GetDoubleValue(v, i) * coefficients[v]);
+    }
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/M5regLeaf.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/M5regLeaf.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/LeafTypes/M5regLeaf.cs	(revision 17078)
@@ -0,0 +1,64 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Algorithms.DataAnalysis.Glmnet;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("0AED959D-78C3-4927-BDCF-473D0AEE32AA")]
+  [Item("M5regLeaf", "A leaf type that uses regularized linear models as leaf models.")]
+  public sealed class M5regLeaf : LeafBase {
+    #region Constructors & Cloning
+    [StorableConstructor]
+    private M5regLeaf(StorableConstructorFlag _) : base(_) { }
+    private M5regLeaf(M5regLeaf original, Cloner cloner) : base(original, cloner) { }
+    public M5regLeaf() { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new M5regLeaf(this, cloner);
+    }
+    #endregion
+
+    #region IModelType
+    public override bool ProvidesConfidence {
+      get { return false; }
+    }
+
+    public override IRegressionModel Build(IRegressionProblemData pd, IRandom random, CancellationToken cancellationToken, out int numberOfParameters) {
+      if (pd.Dataset.Rows < MinLeafSize(pd)) throw new ArgumentException("The number of training instances is too small to create a linear model");
+      numberOfParameters = pd.AllowedInputVariables.Count() + 1;
+
+      double x1, x2;
+      var coeffs = ElasticNetLinearRegression.CalculateModelCoefficients(pd, 1, 0.2, out x1, out x2);
+      numberOfParameters = coeffs.Length;
+      return ElasticNetLinearRegression.CreateSymbolicSolution(coeffs, pd).Model;
+    }
+    public override int MinLeafSize(IRegressionProblemData pd) {
+      return pd.AllowedInputVariables.Count() + 2;
+    }
+    #endregion
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/M5Regression.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/M5Regression.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/M5Regression.cs	(revision 17078)
@@ -0,0 +1,306 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+ using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.PermutationEncoding;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.PluginInfrastructure;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Random;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("FC8D8E5A-D16D-41BB-91CF-B2B35D17ADD7")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 95)]
+  [Item("M5RegressionTree", "A M5 regression tree / rule set")]
+  public sealed class M5Regression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+    public override bool SupportsPause {
+      get { return true; }
+    }
+
+    public const string RegressionTreeParameterVariableName = "RegressionTreeParameters";
+    public const string ModelVariableName = "Model";
+    public const string PruningSetVariableName = "PruningSet";
+    public const string TrainingSetVariableName = "TrainingSet";
+
+    #region Parameter names
+    private const string GenerateRulesParameterName = "GenerateRules";
+    private const string HoldoutSizeParameterName = "HoldoutSize";
+    private const string SplitterParameterName = "Splitter";
+    private const string MinimalNodeSizeParameterName = "MinimalNodeSize";
+    private const string LeafModelParameterName = "LeafModel";
+    private const string PruningTypeParameterName = "PruningType";
+    private const string SeedParameterName = "Seed";
+    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+    private const string UseHoldoutParameterName = "UseHoldout";
+    #endregion
+
+    #region Parameter properties
+    public IFixedValueParameter<BoolValue> GenerateRulesParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[GenerateRulesParameterName]; }
+    }
+    public IFixedValueParameter<PercentValue> HoldoutSizeParameter {
+      get { return (IFixedValueParameter<PercentValue>)Parameters[HoldoutSizeParameterName]; }
+    }
+    public IConstrainedValueParameter<ISplitter> SplitterParameter {
+      get { return (IConstrainedValueParameter<ISplitter>)Parameters[SplitterParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> MinimalNodeSizeParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[MinimalNodeSizeParameterName]; }
+    }
+    public IConstrainedValueParameter<ILeafModel> LeafModelParameter {
+      get { return (IConstrainedValueParameter<ILeafModel>)Parameters[LeafModelParameterName]; }
+    }
+    public IConstrainedValueParameter<IPruning> PruningTypeParameter {
+      get { return (IConstrainedValueParameter<IPruning>)Parameters[PruningTypeParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> SeedParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> UseHoldoutParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[UseHoldoutParameterName]; }
+    }
+    #endregion
+
+    #region Properties 
+    public bool GenerateRules {
+      get { return GenerateRulesParameter.Value.Value; }
+      set { GenerateRulesParameter.Value.Value = value; }
+    }
+    public double HoldoutSize {
+      get { return HoldoutSizeParameter.Value.Value; }
+      set { HoldoutSizeParameter.Value.Value = value; }
+    }
+    public ISplitter Splitter {
+      get { return SplitterParameter.Value; }
+      // no setter because this is a constrained parameter
+    }
+    public int MinimalNodeSize {
+      get { return MinimalNodeSizeParameter.Value.Value; }
+      set { MinimalNodeSizeParameter.Value.Value = value; }
+    }
+    public ILeafModel LeafModel {
+      get { return LeafModelParameter.Value; }
+    }
+    public IPruning Pruning {
+      get { return PruningTypeParameter.Value; }
+    }
+    public int Seed {
+      get { return SeedParameter.Value.Value; }
+      set { SeedParameter.Value.Value = value; }
+    }
+    public bool SetSeedRandomly {
+      get { return SetSeedRandomlyParameter.Value.Value; }
+      set { SetSeedRandomlyParameter.Value.Value = value; }
+    }
+    public bool UseHoldout {
+      get { return UseHoldoutParameter.Value.Value; }
+      set { UseHoldoutParameter.Value.Value = value; }
+    }
+    #endregion
+
+    #region State
+    [Storable]
+    private IScope stateScope;
+    #endregion
+
+    #region Constructors and Cloning
+    [StorableConstructor]
+    private M5Regression(StorableConstructorFlag _) : base(_) { }
+    private M5Regression(M5Regression original, Cloner cloner) : base(original, cloner) {
+      stateScope = cloner.Clone(stateScope);
+    }
+    public M5Regression() {
+      var modelSet = new ItemSet<ILeafModel>(ApplicationManager.Manager.GetInstances<ILeafModel>());
+      var pruningSet = new ItemSet<IPruning>(ApplicationManager.Manager.GetInstances<IPruning>());
+      var splitterSet = new ItemSet<ISplitter>(ApplicationManager.Manager.GetInstances<ISplitter>());
+      Parameters.Add(new FixedValueParameter<BoolValue>(GenerateRulesParameterName, "Whether a set of rules or a decision tree shall be created (default=false)", new BoolValue(false)));
+      Parameters.Add(new FixedValueParameter<PercentValue>(HoldoutSizeParameterName, "How much of the training set shall be reserved for pruning (default=20%).", new PercentValue(0.2)));
+      Parameters.Add(new ConstrainedValueParameter<ISplitter>(SplitterParameterName, "The type of split function used to create node splits (default='M5Splitter').", splitterSet, splitterSet.OfType<M5Splitter>().First()));
+      Parameters.Add(new FixedValueParameter<IntValue>(MinimalNodeSizeParameterName, "The minimal number of samples in a leaf node (default=1).", new IntValue(1)));
+      Parameters.Add(new ConstrainedValueParameter<ILeafModel>(LeafModelParameterName, "The type of model used for the nodes (default='LinearLeaf').", modelSet, modelSet.OfType<LinearLeaf>().First()));
+      Parameters.Add(new ConstrainedValueParameter<IPruning>(PruningTypeParameterName, "The type of pruning used (default='ComplexityPruning').", pruningSet, pruningSet.OfType<ComplexityPruning>().First()));
+      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(UseHoldoutParameterName, "True if a holdout set should be generated, false if splitting and pruning shall be performed on the same data (default=false).", new BoolValue(false)));
+      Problem = new RegressionProblem();
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new M5Regression(this, cloner);
+    }
+    #endregion
+
+    protected override void Initialize(CancellationToken cancellationToken) {
+      base.Initialize(cancellationToken);
+      var random = new MersenneTwister();
+      if (SetSeedRandomly) Seed = RandomSeedGenerator.GetSeed();
+      random.Reset(Seed);
+      stateScope = InitializeScope(random, Problem.ProblemData, Pruning, MinimalNodeSize, LeafModel, Splitter, GenerateRules, UseHoldout, HoldoutSize);
+      stateScope.Variables.Add(new Variable("Algorithm", this));
+      Results.AddOrUpdateResult("StateScope", stateScope);
+    }
+
+    protected override void Run(CancellationToken cancellationToken) {
+      var model = Build(stateScope, Results, cancellationToken);
+      AnalyzeSolution(model.CreateRegressionSolution(Problem.ProblemData), Results, Problem.ProblemData);
+    }
+
+    #region Static Interface
+    public static IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData, IRandom random, ILeafModel leafModel = null, ISplitter splitter = null, IPruning pruning = null,
+      bool useHoldout = false, double holdoutSize = 0.2, int minimumLeafSize = 1, bool generateRules = false, ResultCollection results = null, CancellationToken? cancellationToken = null) {
+      if (leafModel == null) leafModel = new LinearLeaf();
+      if (splitter == null) splitter = new M5Splitter();
+      if (cancellationToken == null) cancellationToken = CancellationToken.None;
+      if (pruning == null) pruning = new ComplexityPruning();
+
+      var stateScope = InitializeScope(random, problemData, pruning, minimumLeafSize, leafModel, splitter, generateRules, useHoldout, holdoutSize);
+      var model = Build(stateScope, results, cancellationToken.Value);
+      return model.CreateRegressionSolution(problemData);
+    }
+
+    public static void UpdateModel(IM5Model model, IRegressionProblemData problemData, IRandom random, ILeafModel leafModel, CancellationToken? cancellationToken = null) {
+      if (cancellationToken == null) cancellationToken = CancellationToken.None;
+      var regressionTreeParameters = new RegressionTreeParameters(leafModel, problemData, random);
+      var scope = new Scope();
+      scope.Variables.Add(new Variable(RegressionTreeParameterVariableName, regressionTreeParameters));
+      leafModel.Initialize(scope);
+      model.Update(problemData.TrainingIndices.ToList(), scope, cancellationToken.Value);
+    }
+    #endregion
+
+    #region Helpers
+    private static IScope InitializeScope(IRandom random, IRegressionProblemData problemData, IPruning pruning, int minLeafSize, ILeafModel leafModel, ISplitter splitter, bool generateRules, bool useHoldout, double holdoutSize) {
+      var stateScope = new Scope("RegressionTreeStateScope");
+
+      //reduce RegressionProblemData to AllowedInput & Target column wise and to TrainingSet row wise
+      var doubleVars = new HashSet<string>(problemData.Dataset.DoubleVariables);
+      var vars = problemData.AllowedInputVariables.Concat(new[] {problemData.TargetVariable}).ToArray();
+      if (vars.Any(v => !doubleVars.Contains(v))) throw new NotSupportedException("M5 regression supports only double valued input or output features.");
+      var doubles = vars.Select(v => problemData.Dataset.GetDoubleValues(v, problemData.TrainingIndices).ToArray()).ToArray();
+      if (doubles.Any(v => v.Any(x => double.IsNaN(x) || double.IsInfinity(x))))
+        throw new NotSupportedException("M5 regression does not support NaN or infinity values in the input dataset.");
+      var trainingData = new Dataset(vars, doubles);
+      var pd = new RegressionProblemData(trainingData, problemData.AllowedInputVariables, problemData.TargetVariable);
+      pd.TrainingPartition.End = pd.TestPartition.Start = pd.TestPartition.End = pd.Dataset.Rows;
+      pd.TrainingPartition.Start = 0;
+
+      //store regression tree parameters
+      var regressionTreeParams = new RegressionTreeParameters(pruning, minLeafSize, leafModel, pd, random, splitter);
+      stateScope.Variables.Add(new Variable(RegressionTreeParameterVariableName, regressionTreeParams));
+
+      //initialize tree operators
+      pruning.Initialize(stateScope);
+      splitter.Initialize(stateScope);
+      leafModel.Initialize(stateScope);
+
+      //store unbuilt model
+      IItem model;
+      if (generateRules) {
+        model = RegressionRuleSetModel.CreateRuleModel(problemData.TargetVariable, regressionTreeParams);
+        RegressionRuleSetModel.Initialize(stateScope);
+      }
+      else {
+        model = RegressionNodeTreeModel.CreateTreeModel(problemData.TargetVariable, regressionTreeParams);
+      }
+      stateScope.Variables.Add(new Variable(ModelVariableName, model));
+
+      //store training & pruning indices
+      IReadOnlyList<int> trainingSet, pruningSet;
+      GeneratePruningSet(pd.TrainingIndices.ToArray(), random, useHoldout, holdoutSize, out trainingSet, out pruningSet);
+      stateScope.Variables.Add(new Variable(TrainingSetVariableName, new IntArray(trainingSet.ToArray())));
+      stateScope.Variables.Add(new Variable(PruningSetVariableName, new IntArray(pruningSet.ToArray())));
+
+      return stateScope;
+    }
+
+    private static IRegressionModel Build(IScope stateScope, ResultCollection results, CancellationToken cancellationToken) {
+      var regressionTreeParams = (RegressionTreeParameters)stateScope.Variables[RegressionTreeParameterVariableName].Value;
+      var model = (IM5Model)stateScope.Variables[ModelVariableName].Value;
+      var trainingRows = (IntArray)stateScope.Variables[TrainingSetVariableName].Value;
+      var pruningRows = (IntArray)stateScope.Variables[PruningSetVariableName].Value;
+      if (1 > trainingRows.Length)
+        return new PreconstructedLinearModel(new Dictionary<string, double>(), 0, regressionTreeParams.TargetVariable);
+      if (regressionTreeParams.MinLeafSize > trainingRows.Length) {
+        var targets = regressionTreeParams.Data.GetDoubleValues(regressionTreeParams.TargetVariable).ToArray();
+        return new PreconstructedLinearModel(new Dictionary<string, double>(), targets.Average(), regressionTreeParams.TargetVariable);
+      }
+      model.Build(trainingRows.ToArray(), pruningRows.ToArray(), stateScope, results, cancellationToken);
+      return model;
+    }
+
+    private static void GeneratePruningSet(IReadOnlyList<int> allrows, IRandom random, bool useHoldout, double holdoutSize, out IReadOnlyList<int> training, out IReadOnlyList<int> pruning) {
+      if (!useHoldout) {
+        training = allrows;
+        pruning = allrows;
+        return;
+      }
+      var perm = new Permutation(PermutationTypes.Absolute, allrows.Count, random);
+      var cut = (int)(holdoutSize * allrows.Count);
+      pruning = perm.Take(cut).Select(i => allrows[i]).ToArray();
+      training = perm.Take(cut).Select(i => allrows[i]).ToArray();
+    }
+
+    private void AnalyzeSolution(IRegressionSolution solution, ResultCollection results, IRegressionProblemData problemData) {
+      results.Add(new Result("RegressionSolution", (IItem)solution.Clone()));
+
+      Dictionary<string, int> frequencies = null;
+
+      var tree = solution.Model as RegressionNodeTreeModel;
+      if (tree != null) {
+        results.Add(RegressionTreeAnalyzer.CreateLeafDepthHistogram(tree));
+        frequencies = RegressionTreeAnalyzer.GetTreeVariableFrequences(tree);
+        RegressionTreeAnalyzer.AnalyzeNodes(tree, results, problemData);
+      }
+
+      var ruleSet = solution.Model as RegressionRuleSetModel;
+      if (ruleSet != null) {
+        results.Add(RegressionTreeAnalyzer.CreateRulesResult(ruleSet, problemData, "M5Rules", true));
+        frequencies = RegressionTreeAnalyzer.GetRuleVariableFrequences(ruleSet);
+        results.Add(RegressionTreeAnalyzer.CreateCoverageDiagram(ruleSet, problemData));
+      }
+
+      //Variable frequencies
+      if (frequencies != null) {
+        var sum = frequencies.Values.Sum();
+        sum = sum == 0 ? 1 : sum;
+        var impactArray = new DoubleArray(frequencies.Select(i => (double)i.Value / sum).ToArray()) {
+          ElementNames = frequencies.Select(i => i.Key)
+        };
+        results.Add(new Result("Variable Frequences", "relative frequencies of variables in rules and tree nodes", impactArray));
+      }
+
+      var pruning = Pruning as ComplexityPruning;
+      if (pruning != null && tree != null)
+        RegressionTreeAnalyzer.PruningChart(tree, pruning, results);
+    }
+    #endregion
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/M5Utilities/PrincipleComponentTransformation.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/M5Utilities/PrincipleComponentTransformation.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/M5Utilities/PrincipleComponentTransformation.cs	(revision 17078)
@@ -0,0 +1,200 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("A2DDC528-BAA7-445F-98E1-5F895CE2FD5C")]
+  public class PrincipleComponentTransformation : IDeepCloneable {
+    #region Properties
+    [Storable]
+    private double[,] Matrix { get; set; }
+    [Storable]
+    public double[] Variances { get; private set; }
+    [Storable]
+    public string[] VariableNames { get; private set; }
+    [Storable]
+    private double[] Deviations { get; set; }
+    [Storable]
+    private double[] Means { get; set; }
+    public string[] ComponentNames {
+      get { return VariableNames.Select((_, x) => "pc" + x).ToArray(); }
+    }
+    #endregion
+
+    #region HLConstructors
+    [StorableConstructor]
+    protected PrincipleComponentTransformation(StorableConstructorFlag _) { }
+    protected PrincipleComponentTransformation(PrincipleComponentTransformation original, Cloner cloner) {
+      if (original.Variances != null) Variances = original.Variances.ToArray();
+      if (original.VariableNames != null) VariableNames = original.VariableNames.ToArray();
+      if (original.Deviations != null) Deviations = original.Deviations.ToArray();
+      if (original.Means != null) Means = original.Means.ToArray();
+      if (original.Matrix == null) return;
+      Matrix = new double[original.Matrix.GetLength(0), original.Matrix.GetLength(1)];
+      for (var i = 0; i < original.Matrix.GetLength(0); i++)
+      for (var j = 0; j < original.Matrix.GetLength(1); j++)
+        Matrix[i, j] = original.Matrix[i, j];
+    }
+    private PrincipleComponentTransformation() { }
+    public IDeepCloneable Clone(Cloner cloner) {
+      return new PrincipleComponentTransformation(this, cloner);
+    }
+    public object Clone() {
+      return new Cloner().Clone(this);
+    }
+    #endregion
+
+    #region Static Interface
+    public static PrincipleComponentTransformation CreateProjection(IDataset dataset, IEnumerable<int> rows, IEnumerable<string> variables, bool normalize = false) {
+      var res = new PrincipleComponentTransformation();
+      res.BuildPca(dataset, rows, variables, normalize);
+      return res;
+    }
+    #endregion
+
+    #region Projection
+    public IRegressionProblemData TransformProblemData(IRegressionProblemData pd) {
+      return CreateProblemData(pd, TransformDataset(pd.Dataset), ComponentNames);
+    }
+
+    public IDataset TransformDataset(IDataset data) {
+      return CreateDataset(data, TransformData(data, Enumerable.Range(0, data.Rows)));
+    }
+
+    public double[,] TransformData(IDataset dataset, IEnumerable<int> rows) {
+      var instances = rows.ToArray();
+      var result = new double[instances.Length, VariableNames.Length];
+      for (var r = 0; r < instances.Length; r++)
+      for (var i = 0; i < VariableNames.Length; i++) {
+        var val = (dataset.GetDoubleValue(VariableNames[i], instances[r]) - Means[i]) / Deviations[i];
+        for (var j = 0; j < VariableNames.Length; j++)
+          result[r, j] += val * Matrix[i, j];
+      }
+      return result;
+    }
+    #endregion
+
+    #region Reversion
+    public IRegressionProblemData RevertProblemData(IRegressionProblemData pd) {
+      return CreateProblemData(pd, RevertDataset(pd.Dataset), VariableNames);
+    }
+
+    public IDataset RevertDataset(IDataset data) {
+      return CreateRevertedDataset(data, RevertData(data, Enumerable.Range(0, data.Rows)));
+    }
+
+    public double[,] RevertData(IDataset dataset, IEnumerable<int> rows) {
+      var instances = rows.ToArray();
+      var components = ComponentNames;
+      var result = new double[instances.Length, VariableNames.Length];
+      for (var r = 0; r < instances.Length; r++)
+      for (var i = 0; i < components.Length; i++) {
+        var val = dataset.GetDoubleValue(components[i], instances[r]);
+        for (var j = 0; j < VariableNames.Length; j++)
+          result[r, j] += val * Matrix[j, i];
+      }
+      for (var r = 0; r < instances.Length; r++) {
+        for (var j = 0; j < VariableNames.Length; j++) {
+          result[r, j] *= Deviations[j];
+          result[r, j] += Means[j];
+        }
+      }
+
+      return result;
+    }
+    #endregion
+
+    #region Helpers
+    private static IRegressionProblemData CreateProblemData(IRegressionProblemData pd, IDataset data, IReadOnlyList<string> allowedNames) {
+      var res = new RegressionProblemData(data, allowedNames, pd.TargetVariable);
+      res.TestPartition.Start = pd.TestPartition.Start;
+      res.TestPartition.End = pd.TestPartition.End;
+      res.TrainingPartition.Start = pd.TrainingPartition.Start;
+      res.TrainingPartition.End = pd.TrainingPartition.End;
+      res.Name = pd.Name;
+      return res;
+    }
+
+    private IDataset CreateDataset(IDataset data, double[,] pcs) {
+      var n = ComponentNames;
+      var nDouble = data.DoubleVariables.Where(x => !VariableNames.Contains(x)).ToArray();
+      var nDateTime = data.DateTimeVariables.ToArray();
+      var nString = data.StringVariables.ToArray();
+
+      IEnumerable<IList> nData = n.Select((_, x) => Enumerable.Range(0, pcs.GetLength(0)).Select(r => pcs[r, x]).ToList());
+      IEnumerable<IList> nDoubleData = nDouble.Select(x => data.GetDoubleValues(x).ToList());
+      IEnumerable<IList> nDateTimeData = nDateTime.Select(x => data.GetDateTimeValues(x).ToList());
+      IEnumerable<IList> nStringData = nString.Select(x => data.GetStringValues(x).ToList());
+
+      return new Dataset(n.Concat(nDouble).Concat(nDateTime).Concat(nString), nData.Concat(nDoubleData).Concat(nDateTimeData).Concat(nStringData).ToArray());
+    }
+
+    private IDataset CreateRevertedDataset(IDataset data, double[,] pcs) {
+      var n = VariableNames;
+      var nDouble = data.DoubleVariables.Where(x => !ComponentNames.Contains(x)).ToArray();
+      var nDateTime = data.DateTimeVariables.ToArray();
+      var nString = data.StringVariables.ToArray();
+
+      IEnumerable<IList> nData = n.Select((_, x) => Enumerable.Range(0, pcs.GetLength(0)).Select(r => pcs[r, x]).ToList());
+      IEnumerable<IList> nDoubleData = nDouble.Select(x => data.GetDoubleValues(x).ToList());
+      IEnumerable<IList> nDateTimeData = nDateTime.Select(x => data.GetDateTimeValues(x).ToList());
+      IEnumerable<IList> nStringData = nString.Select(x => data.GetStringValues(x).ToList());
+
+      return new Dataset(n.Concat(nDouble).Concat(nDateTime).Concat(nString), nData.Concat(nDoubleData).Concat(nDateTimeData).Concat(nStringData).ToArray());
+    }
+
+    private void BuildPca(IDataset dataset, IEnumerable<int> rows, IEnumerable<string> variables, bool normalize) {
+      var instances = rows.ToArray();
+      var attributes = variables.ToArray();
+      Means = normalize
+        ? attributes.Select(v => dataset.GetDoubleValues(v, instances).Average()).ToArray()
+        : attributes.Select(x => 0.0).ToArray();
+      Deviations = normalize
+        ? attributes.Select(v => dataset.GetDoubleValues(v, instances).StandardDeviationPop()).Select(x => x.IsAlmost(0.0) ? 1 : x).ToArray()
+        : attributes.Select(x => 1.0).ToArray();
+
+      var data = new double[instances.Length, attributes.Length];
+
+      for (var j = 0; j < attributes.Length; j++) {
+        var i = 0;
+        foreach (var v in dataset.GetDoubleValues(attributes[j], instances)) {
+          data[i, j] = (v - Means[j]) / Deviations[j];
+          i++;
+        }
+      }
+
+      int info;
+      double[] variances;
+      double[,] matrix;
+      alglib.pcabuildbasis(data, instances.Length, attributes.Length, out info, out variances, out matrix);
+      Matrix = matrix;
+      Variances = variances;
+      VariableNames = attributes;
+    }
+    #endregion
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/M5Utilities/RegressionTreeAnalyzer.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/M5Utilities/RegressionTreeAnalyzer.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/M5Utilities/RegressionTreeAnalyzer.cs	(revision 17078)
@@ -0,0 +1,266 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Analysis;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public static class RegressionTreeAnalyzer {
+    private const string ConditionResultName = "Condition";
+    private const string CoverResultName = "Covered Instances";
+    private const string CoverageDiagramResultName = "Coverage";
+    private const string RuleModelResultName = "RuleModel";
+
+    public static Dictionary<string, int> GetRuleVariableFrequences(RegressionRuleSetModel ruleSetModel) {
+      var res = ruleSetModel.VariablesUsedForPrediction.ToDictionary(x => x, x => 0);
+      foreach (var rule in ruleSetModel.Rules)
+      foreach (var att in rule.SplitAttributes)
+        res[att]++;
+      return res;
+    }
+
+    public static Dictionary<string, int> GetTreeVariableFrequences(RegressionNodeTreeModel treeModel) {
+      var res = treeModel.VariablesUsedForPrediction.ToDictionary(x => x, x => 0);
+      var root = treeModel.Root;
+      foreach (var cur in root.EnumerateNodes().Where(x => !x.IsLeaf))
+        res[cur.SplitAttribute]++;
+      return res;
+    }
+
+    public static Result CreateLeafDepthHistogram(RegressionNodeTreeModel treeModel) {
+      var list = new List<int>();
+      GetLeafDepths(treeModel.Root, 0, list);
+      var row = new DataRow("Depths", "", list.Select(x => (double)x)) {
+        VisualProperties = {ChartType = DataRowVisualProperties.DataRowChartType.Histogram}
+      };
+      var hist = new DataTable("LeafDepths");
+      hist.Rows.Add(row);
+      return new Result(hist.Name, hist);
+    }
+
+    public static Result CreateRulesResult(RegressionRuleSetModel ruleSetModel, IRegressionProblemData pd, string resultName, bool displayModels) {
+      var res = new ResultCollection();
+      var i = 0;
+      foreach (var rule in ruleSetModel.Rules)
+        res.Add(new Result("Rule" + i++, CreateRulesResult(rule, pd, displayModels, out pd)));
+      return new Result(resultName, res);
+    }
+
+    public static IResult CreateCoverageDiagram(RegressionRuleSetModel setModel, IRegressionProblemData problemData) {
+      var res = new DataTable(CoverageDiagramResultName);
+      var training = CountCoverage(setModel, problemData.Dataset, problemData.TrainingIndices);
+      var test = CountCoverage(setModel, problemData.Dataset, problemData.TestIndices);
+      res.Rows.Add(new DataRow("Training", "", training));
+      res.Rows.Add(new DataRow("Test", "", test));
+
+      foreach (var row in res.Rows)
+        row.VisualProperties.ChartType = DataRowVisualProperties.DataRowChartType.Columns;
+      res.VisualProperties.XAxisMaximumFixedValue = training.Count + 1;
+      res.VisualProperties.XAxisMaximumAuto = false;
+      res.VisualProperties.XAxisMinimumFixedValue = 0;
+      res.VisualProperties.XAxisMinimumAuto = false;
+      res.VisualProperties.XAxisTitle = "Rule";
+      res.VisualProperties.YAxisTitle = "Covered Instances";
+
+      return new Result(CoverageDiagramResultName, res);
+    }
+
+    private static void GetLeafDepths(RegressionNodeModel n, int depth, ICollection<int> res) {
+      if (n == null) return;
+      if (n.Left == null && n.Right == null) res.Add(depth);
+      else {
+        GetLeafDepths(n.Left, depth + 1, res);
+        GetLeafDepths(n.Right, depth + 1, res);
+      }
+    }
+
+    private static IScope CreateRulesResult(RegressionRuleModel regressionRuleModel, IRegressionProblemData pd, bool displayModels, out IRegressionProblemData notCovered) {
+      var training = pd.TrainingIndices.Where(x => !regressionRuleModel.Covers(pd.Dataset, x)).ToArray();
+      var test = pd.TestIndices.Where(x => !regressionRuleModel.Covers(pd.Dataset, x)).ToArray();
+      if (training.Length > 0 || test.Length > 0) {
+        var data = new Dataset(pd.Dataset.DoubleVariables, pd.Dataset.DoubleVariables.Select(v => pd.Dataset.GetDoubleValues(v, training.Concat(test)).ToArray()));
+        notCovered = new RegressionProblemData(data, pd.AllowedInputVariables, pd.TargetVariable);
+        notCovered.TestPartition.Start = notCovered.TrainingPartition.End = training.Length;
+        notCovered.TestPartition.End = training.Length + test.Length;
+      } else notCovered = null;
+
+      var training2 = pd.TrainingIndices.Where(x => regressionRuleModel.Covers(pd.Dataset, x)).ToArray();
+      var test2 = pd.TestIndices.Where(x => regressionRuleModel.Covers(pd.Dataset, x)).ToArray();
+      var data2 = new Dataset(pd.Dataset.DoubleVariables, pd.Dataset.DoubleVariables.Select(v => pd.Dataset.GetDoubleValues(v, training2.Concat(test2)).ToArray()));
+      var covered = new RegressionProblemData(data2, pd.AllowedInputVariables, pd.TargetVariable);
+      covered.TestPartition.Start = covered.TrainingPartition.End = training2.Length;
+      covered.TestPartition.End = training2.Length + test2.Length;
+
+      var res2 = new Scope("RuleModels");
+      res2.Variables.Add(new Variable(ConditionResultName, new StringValue(regressionRuleModel.ToCompactString())));
+      res2.Variables.Add(new Variable(CoverResultName, new IntValue(pd.TrainingIndices.Count() - training.Length)));
+      if (displayModels)
+        res2.Variables.Add(new Variable(RuleModelResultName, regressionRuleModel.CreateRegressionSolution(covered)));
+      return res2;
+    }
+
+    private static IReadOnlyList<double> CountCoverage(RegressionRuleSetModel setModel, IDataset data, IEnumerable<int> rows) {
+      var rules = setModel.Rules.ToArray();
+      var res = new double[rules.Length];
+      foreach (var row in rows)
+        for (var i = 0; i < rules.Length; i++)
+          if (rules[i].Covers(data, row)) {
+            res[i]++;
+            break;
+          }
+      return res;
+    }
+
+    public static void AnalyzeNodes(RegressionNodeTreeModel tree, ResultCollection results, IRegressionProblemData pd) {
+      var dict = new Dictionary<int, RegressionNodeModel>();
+      var trainingLeafRows = new Dictionary<int, IReadOnlyList<int>>();
+      var testLeafRows = new Dictionary<int, IReadOnlyList<int>>();
+      var modelNumber = new IntValue(1);
+      var symtree = new SymbolicExpressionTree(MirrorTree(tree.Root, dict, trainingLeafRows, testLeafRows, modelNumber, pd.Dataset, pd.TrainingIndices.ToList(), pd.TestIndices.ToList()));
+      results.AddOrUpdateResult("DecisionTree", symtree);
+
+      if (dict.Count > 200) return;
+      var models = new Scope("NodeModels");
+      results.AddOrUpdateResult("NodeModels", models);
+      foreach (var m in dict.Keys.OrderBy(x => x))
+        models.Variables.Add(new Variable("Model " + m, dict[m].CreateRegressionSolution(Subselect(pd, trainingLeafRows[m], testLeafRows[m]))));
+    }
+
+    public static void PruningChart(RegressionNodeTreeModel tree, ComplexityPruning pruning, ResultCollection results) {
+      var nodes = new Queue<RegressionNodeModel>();
+      nodes.Enqueue(tree.Root);
+      var max = 0.0;
+      var strenghts = new SortedList<double, int>();
+      while (nodes.Count > 0) {
+        var n = nodes.Dequeue();
+
+        if (n.IsLeaf) {
+          max++;
+          continue;
+        }
+
+        if (!strenghts.ContainsKey(n.PruningStrength)) strenghts.Add(n.PruningStrength, 0);
+        strenghts[n.PruningStrength]++;
+        nodes.Enqueue(n.Left);
+        nodes.Enqueue(n.Right);
+      }
+      if (strenghts.Count == 0) return;
+
+      var plot = new ScatterPlot("Pruned Sizes", "") {
+        VisualProperties = {
+          XAxisTitle = "Pruning Strength",
+          YAxisTitle = "Tree Size",
+          XAxisMinimumAuto = false,
+          XAxisMinimumFixedValue = 0
+        }
+      };
+      var row = new ScatterPlotDataRow("TreeSizes", "", new List<Point2D<double>>());
+      row.Points.Add(new Point2D<double>(pruning.PruningStrength, max));
+
+      var fillerDots = new Queue<double>();
+      var minX = pruning.PruningStrength;
+      var maxX = strenghts.Last().Key;
+      var size = (maxX - minX) / 200;
+      for (var x = minX; x <= maxX; x += size) {
+        fillerDots.Enqueue(x);
+      }
+
+      foreach (var strenght in strenghts.Keys) {
+        while (fillerDots.Count > 0 && strenght > fillerDots.Peek())
+          row.Points.Add(new Point2D<double>(fillerDots.Dequeue(), max));
+        max -= strenghts[strenght];
+        row.Points.Add(new Point2D<double>(strenght, max));
+      }
+
+
+      row.VisualProperties.PointSize = 6;
+      plot.Rows.Add(row);
+      results.AddOrUpdateResult("PruningSizes", plot);
+    }
+
+
+    private static IRegressionProblemData Subselect(IRegressionProblemData data, IReadOnlyList<int> training, IReadOnlyList<int> test) {
+      var dataset = RegressionTreeUtilities.ReduceDataset(data.Dataset, training.Concat(test).ToList(), data.AllowedInputVariables.ToList(), data.TargetVariable);
+      var res = new RegressionProblemData(dataset, data.AllowedInputVariables, data.TargetVariable);
+      res.TrainingPartition.Start = 0;
+      res.TrainingPartition.End = training.Count;
+      res.TestPartition.Start = training.Count;
+      res.TestPartition.End = training.Count + test.Count;
+      return res;
+    }
+
+    private static SymbolicExpressionTreeNode MirrorTree(RegressionNodeModel regressionNode, IDictionary<int, RegressionNodeModel> dict,
+      IDictionary<int, IReadOnlyList<int>> trainingLeafRows,
+      IDictionary<int, IReadOnlyList<int>> testLeafRows,
+      IntValue nextId, IDataset data, IReadOnlyList<int> trainingRows, IReadOnlyList<int> testRows) {
+      if (regressionNode.IsLeaf) {
+        var i = nextId.Value++;
+        dict.Add(i, regressionNode);
+        trainingLeafRows.Add(i, trainingRows);
+        testLeafRows.Add(i, testRows);
+        return new SymbolicExpressionTreeNode(new TextSymbol("Model " + i + "\n(" + trainingRows.Count + "/" + testRows.Count + ")"));
+      }
+
+      var pftext = "\npf = " + regressionNode.PruningStrength.ToString("0.###");
+      var text = regressionNode.SplitAttribute + " <= " + regressionNode.SplitValue.ToString("0.###");
+      if (!double.IsNaN(regressionNode.PruningStrength)) text += pftext;
+
+      var textNode = new SymbolicExpressionTreeNode(new TextSymbol(text));
+      IReadOnlyList<int> lTrainingRows, rTrainingRows;
+      IReadOnlyList<int> lTestRows, rTestRows;
+      RegressionTreeUtilities.SplitRows(trainingRows, data, regressionNode.SplitAttribute, regressionNode.SplitValue, out lTrainingRows, out rTrainingRows);
+      RegressionTreeUtilities.SplitRows(testRows, data, regressionNode.SplitAttribute, regressionNode.SplitValue, out lTestRows, out rTestRows);
+
+      textNode.AddSubtree(MirrorTree(regressionNode.Left, dict, trainingLeafRows, testLeafRows, nextId, data, lTrainingRows, lTestRows));
+      textNode.AddSubtree(MirrorTree(regressionNode.Right, dict, trainingLeafRows, testLeafRows, nextId, data, rTrainingRows, rTestRows));
+
+      return textNode;
+    }
+
+
+    [StorableType("D5540C63-310B-4D6F-8A3D-6C1A08DE7F80")]
+    private sealed class TextSymbol : Symbol {
+      [StorableConstructor]
+      private TextSymbol(StorableConstructorFlag _) : base(_) { }
+      private TextSymbol(Symbol original, Cloner cloner) : base(original, cloner) { }
+      public TextSymbol(string name) : base(name, "") {
+        Name = name;
+      }
+      public override IDeepCloneable Clone(Cloner cloner) {
+        return new TextSymbol(this, cloner);
+      }
+      public override int MinimumArity {
+        get { return 0; }
+      }
+      public override int MaximumArity {
+        get { return int.MaxValue; }
+      }
+    }
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/M5Utilities/RegressionTreeParameters.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/M5Utilities/RegressionTreeParameters.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/M5Utilities/RegressionTreeParameters.cs	(revision 17078)
@@ -0,0 +1,97 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("A6293516-C146-469D-B248-31B866A1D94F")]
+  public sealed class RegressionTreeParameters : Item {
+    private readonly ISplitter splitter;
+    private readonly IPruning pruning;
+    private readonly ILeafModel leafModel;
+    private readonly int minLeafSize;
+    private readonly IRegressionProblemData problemData;
+    private readonly IRandom random;
+    public ISplitter Splitter {
+      get { return splitter; }
+    }
+    public IPruning Pruning {
+      get { return pruning; }
+    }
+    public ILeafModel LeafModel {
+      get { return leafModel; }
+    }
+    public int MinLeafSize {
+      get { return minLeafSize; }
+    }
+    private IRegressionProblemData ProblemData {
+      get { return problemData; }
+    }
+    public IRandom Random {
+      get { return random; }
+    }
+    public IEnumerable<string> AllowedInputVariables {
+      get { return ProblemData.AllowedInputVariables; }
+    }
+    public string TargetVariable {
+      get { return ProblemData.TargetVariable; }
+    }
+    public IDataset Data {
+      get { return ProblemData.Dataset; }
+    }
+
+    #region Constructors & Cloning
+    [StorableConstructor]
+    private RegressionTreeParameters(StorableConstructorFlag _) : base(_) { }
+    private RegressionTreeParameters(RegressionTreeParameters original, Cloner cloner) : base(original, cloner) {
+      problemData = cloner.Clone(original.problemData);
+      random = cloner.Clone(original.random);
+      leafModel = cloner.Clone(original.leafModel);
+      splitter = cloner.Clone(original.splitter);
+      pruning = cloner.Clone(original.pruning);
+      minLeafSize = original.minLeafSize;
+    }
+
+    public RegressionTreeParameters(IPruning pruning, int minleafSize, ILeafModel leafModel,
+      IRegressionProblemData problemData, IRandom random, ISplitter splitter) {
+      this.problemData = problemData;
+      this.random = random;
+      this.leafModel = leafModel;
+      this.splitter = splitter;
+      this.pruning = pruning;
+      minLeafSize = Math.Max(pruning.MinLeafSize(problemData, leafModel), Math.Max(minleafSize, leafModel.MinLeafSize(problemData)));
+    }
+    public RegressionTreeParameters(ILeafModel modeltype, IRegressionProblemData problemData, IRandom random) {
+      this.problemData = problemData;
+      this.random = random;
+      leafModel = modeltype;
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RegressionTreeParameters(this, cloner);
+    }
+    #endregion
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/M5Utilities/RegressionTreeUtilities.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/M5Utilities/RegressionTreeUtilities.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/M5Utilities/RegressionTreeUtilities.cs	(revision 17078)
@@ -0,0 +1,61 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  internal static class RegressionTreeUtilities {
+    public static ResultCollection RunSubAlgorithm(IAlgorithm alg, int random, CancellationToken cancellationToken) {
+      if (alg.Parameters.ContainsKey("SetSeedRandomly") && alg.Parameters.ContainsKey("Seed")) {
+        var seed = alg.Parameters["Seed"].ActualValue as IntValue;
+        var setSeed = alg.Parameters["SetSeedRandomly"].ActualValue as BoolValue;
+        if (seed == null || setSeed == null)
+          throw new ArgumentException("The parameters SetSeedRandomly and Seed do not have the expected type");
+        setSeed.Value = false;
+        seed.Value = random;
+      }
+      if (alg.ExecutionState != ExecutionState.Paused) alg.Prepare();
+      alg.Start(cancellationToken);
+      var res = alg.Results;
+      alg.Runs.Clear();
+      return res;
+    }
+
+    public static void SplitRows(IReadOnlyList<int> rows, IDataset data, string splitAttr, double splitValue, out IReadOnlyList<int> leftRows, out IReadOnlyList<int> rightRows) {
+      //TODO check and revert?: points at borders are now used multipe times
+      var assignment = data.GetDoubleValues(splitAttr, rows).Select(x => x.IsAlmost(splitValue) ? 2 : x < splitValue ? 0 : 1).ToArray();
+      leftRows = rows.Zip(assignment, (i, b) => new {i, b}).Where(x => x.b == 0 || x.b == 2).Select(x => x.i).ToList();
+      rightRows = rows.Zip(assignment, (i, b) => new {i, b}).Where(x => x.b > 0).Select(x => x.i).ToList();
+    }
+
+    public static IDataset ReduceDataset(IDataset data, IReadOnlyList<int> rows, IReadOnlyList<string> inputVariables, string target) {
+      return new Dataset(inputVariables.Concat(new[] {target}), inputVariables.Concat(new[] {target}).Select(x => data.GetDoubleValues(x, rows).ToList()));
+    }
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/MetaModels/RegressionNodeModel.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/MetaModels/RegressionNodeModel.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/MetaModels/RegressionNodeModel.cs	(revision 17078)
@@ -0,0 +1,175 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;                                                  
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("C20C7DF1-CE33-4CCD-88D3-E145CFE239AC")]
+  public class RegressionNodeModel : RegressionModel {
+    #region Properties
+    public double PruningStrength = double.NaN;
+    private IReadOnlyList<string> Variables {
+      get {
+        if (IsLeaf && Model == null) return new List<string>();
+        if (IsLeaf) return Model.VariablesUsedForPrediction.ToList();
+        var set = new HashSet<string> {SplitAttribute};
+        var vl = Left.Variables;
+        var vr = Right.Variables;
+        for (var i = 0; i < vl.Count; i++) set.Add(vl[i]);
+        for (var i = 0; i < vr.Count; i++) set.Add(vr[i]);
+        return set.ToList();
+      }
+    }
+    [Storable]
+    internal int NumSamples { get; private set; }
+    [Storable]
+    internal bool IsLeaf { get; private set; }
+    [Storable]
+    private IRegressionModel Model { get; set; }
+
+    [Storable]
+    public string SplitAttribute { get; private set; }
+    [Storable]
+    public double SplitValue { get; private set; }
+    [Storable]
+    public RegressionNodeModel Left { get; private set; }
+    [Storable]
+    public RegressionNodeModel Right { get; private set; }
+    [Storable]
+    public RegressionNodeModel Parent { get; private set; }
+    #endregion
+
+    #region HLConstructors
+    [StorableConstructor]
+    protected RegressionNodeModel(StorableConstructorFlag _) : base(_) { }
+    protected RegressionNodeModel(RegressionNodeModel original, Cloner cloner) : base(original, cloner) {
+      IsLeaf = original.IsLeaf;
+      Model = cloner.Clone(original.Model);
+      SplitValue = original.SplitValue;
+      SplitAttribute = original.SplitAttribute;
+      Left = cloner.Clone(original.Left);
+      Right = cloner.Clone(original.Right);
+      Parent = cloner.Clone(original.Parent);
+      NumSamples = original.NumSamples;
+    }
+    private RegressionNodeModel(string targetAttr) : base(targetAttr) {
+      IsLeaf = true;
+    }
+    private RegressionNodeModel(RegressionNodeModel parent) : this(parent.TargetVariable) {
+      Parent = parent;
+      IsLeaf = true;
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RegressionNodeModel(this, cloner);
+    }
+    public static RegressionNodeModel CreateNode(string targetAttr, RegressionTreeParameters regressionTreeParams) {
+      return regressionTreeParams.LeafModel.ProvidesConfidence ? new ConfidenceRegressionNodeModel(targetAttr) : new RegressionNodeModel(targetAttr);
+    }
+    private static RegressionNodeModel CreateNode(RegressionNodeModel parent, RegressionTreeParameters regressionTreeParams) {
+      return regressionTreeParams.LeafModel.ProvidesConfidence ? new ConfidenceRegressionNodeModel(parent) : new RegressionNodeModel(parent);
+    }
+    #endregion
+
+    #region RegressionModel
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return Variables; }
+    }
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      if (!IsLeaf) return rows.Select(row => GetEstimatedValue(dataset, row));
+      if (Model == null) throw new NotSupportedException("The model has not been built correctly");
+      return Model.GetEstimatedValues(dataset, rows);
+    }
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RegressionSolution(this, problemData);
+    }
+    #endregion
+
+    internal void Split(RegressionTreeParameters regressionTreeParams, string splitAttribute, double splitValue, int numSamples) {
+      NumSamples = numSamples;
+      SplitAttribute = splitAttribute;
+      SplitValue = splitValue;
+      Left = CreateNode(this, regressionTreeParams);
+      Right = CreateNode(this, regressionTreeParams);
+      IsLeaf = false;
+    }
+
+    internal void ToLeaf() {
+      IsLeaf = true;
+      Right = null;
+      Left = null;
+    }
+
+    internal void SetLeafModel(IRegressionModel model) {
+      Model = model;
+    }
+
+    internal IEnumerable<RegressionNodeModel> EnumerateNodes() {
+      var queue = new Queue<RegressionNodeModel>();
+      queue.Enqueue(this);
+      while (queue.Count != 0) {
+        var cur = queue.Dequeue();
+        yield return cur;
+        if (cur.Left == null && cur.Right == null) continue;
+        if (cur.Left != null) queue.Enqueue(cur.Left);
+        if (cur.Right != null) queue.Enqueue(cur.Right);
+      }
+    }
+
+    #region Helpers
+    private double GetEstimatedValue(IDataset dataset, int row) {
+      if (!IsLeaf) return (dataset.GetDoubleValue(SplitAttribute, row) <= SplitValue ? Left : Right).GetEstimatedValue(dataset, row);
+      if (Model == null) throw new NotSupportedException("The model has not been built correctly");
+      return Model.GetEstimatedValues(dataset, new[] {row}).First();
+    }
+    #endregion
+
+    [StorableType("1FF9E216-6AF1-4282-A7EF-3FA0C1DB29C8")]
+    private sealed class ConfidenceRegressionNodeModel : RegressionNodeModel, IConfidenceRegressionModel {
+      #region HLConstructors
+      [StorableConstructor]
+      private ConfidenceRegressionNodeModel(StorableConstructorFlag _) : base(_) { }
+      private ConfidenceRegressionNodeModel(ConfidenceRegressionNodeModel original, Cloner cloner) : base(original, cloner) { }
+      public ConfidenceRegressionNodeModel(string targetAttr) : base(targetAttr) { }
+      public ConfidenceRegressionNodeModel(RegressionNodeModel parent) : base(parent) { }
+      public override IDeepCloneable Clone(Cloner cloner) {
+        return new ConfidenceRegressionNodeModel(this, cloner);
+      }
+      #endregion
+
+      public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
+        return IsLeaf ? ((IConfidenceRegressionModel)Model).GetEstimatedVariances(dataset, rows) : rows.Select(row => GetEstimatedVariance(dataset, row));
+      }
+
+      private double GetEstimatedVariance(IDataset dataset, int row) {
+        return !IsLeaf ? ((IConfidenceRegressionModel)(dataset.GetDoubleValue(SplitAttribute, row) <= SplitValue ? Left : Right)).GetEstimatedVariances(dataset, row.ToEnumerable()).Single() : ((IConfidenceRegressionModel)Model).GetEstimatedVariances(dataset, new[] {row}).First();
+      }
+
+      public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+        return new ConfidenceRegressionSolution(this, problemData);
+      }
+    }
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/MetaModels/RegressionNodeTreeModel.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/MetaModels/RegressionNodeTreeModel.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/MetaModels/RegressionNodeTreeModel.cs	(revision 17078)
@@ -0,0 +1,132 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("FAF1F955-82F3-4824-9759-9D2846E831AE")]
+  public class RegressionNodeTreeModel : RegressionModel, IM5Model {
+    public const string NumCurrentLeafsResultName = "Number of current leafs";
+    public const string RootVariableName = "Root";
+    #region Properties
+    [Storable]
+    internal RegressionNodeModel Root { get; private set; }
+    #endregion
+
+    #region HLConstructors & Cloning
+    [StorableConstructor]
+    protected RegressionNodeTreeModel(StorableConstructorFlag _) : base(_) { }
+    protected RegressionNodeTreeModel(RegressionNodeTreeModel original, Cloner cloner) : base(original, cloner) {
+      Root = cloner.Clone(original.Root);
+    }
+    protected RegressionNodeTreeModel(string targetVariable) : base(targetVariable) { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RegressionNodeTreeModel(this, cloner);
+    }
+    #endregion
+
+    internal static RegressionNodeTreeModel CreateTreeModel(string targetAttr, RegressionTreeParameters regressionTreeParams) {
+      return regressionTreeParams.LeafModel.ProvidesConfidence ? new ConfidenceRegressionNodeTreeModel(targetAttr) : new RegressionNodeTreeModel(targetAttr);
+    }
+
+    #region RegressionModel
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return Root.VariablesUsedForPrediction ?? new List<string>(); }
+    }
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      if (Root == null) throw new NotSupportedException("The model has not been built yet");
+      return Root.GetEstimatedValues(dataset, rows);
+    }
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RegressionSolution(this, problemData);
+    }
+    #endregion
+
+    #region IM5Model
+    public void Build(IReadOnlyList<int> trainingRows, IReadOnlyList<int> pruningRows, IScope statescope, ResultCollection results, CancellationToken cancellationToken) {
+      var regressionTreeParams = (RegressionTreeParameters)statescope.Variables[M5Regression.RegressionTreeParameterVariableName].Value;
+      //start with one node
+      if (Root == null)
+        Root = RegressionNodeModel.CreateNode(regressionTreeParams.TargetVariable, regressionTreeParams);
+
+      //split into (overfitted tree)
+      regressionTreeParams.Splitter.Split(this, trainingRows, statescope, cancellationToken);
+
+      //prune
+      regressionTreeParams.Pruning.Prune(this, trainingRows, pruningRows, statescope, cancellationToken);
+
+      //build final leaf models
+      regressionTreeParams.LeafModel.Build(this, trainingRows.Union(pruningRows).ToArray(), statescope, cancellationToken);
+    }
+
+    public void Update(IReadOnlyList<int> rows, IScope statescope, CancellationToken cancellationToken) {
+      var regressionTreeParams = (RegressionTreeParameters)statescope.Variables[M5Regression.RegressionTreeParameterVariableName].Value;
+      regressionTreeParams.LeafModel.Build(this, rows, statescope, cancellationToken);
+    }
+
+    public static void Initialize(IScope stateScope) {
+      var param = (RegressionTreeParameters)stateScope.Variables[M5Regression.RegressionTreeParameterVariableName].Value;
+      stateScope.Variables.Add(new Variable(RootVariableName, RegressionNodeModel.CreateNode(param.TargetVariable, param)));
+    }
+    #endregion
+
+    public void BuildModel(IReadOnlyList<int> trainingRows, IReadOnlyList<int> pruningRows, IScope statescope, ResultCollection results, CancellationToken cancellationToken) {
+      var regressionTreeParams = (RegressionTreeParameters)statescope.Variables[M5Regression.RegressionTreeParameterVariableName].Value;
+      //start with one node
+      Root = RegressionNodeModel.CreateNode(regressionTreeParams.TargetVariable, regressionTreeParams);
+
+      //split into (overfitted tree)
+      regressionTreeParams.Splitter.Split(this, trainingRows, statescope, cancellationToken);
+
+      //prune
+      regressionTreeParams.Pruning.Prune(this, trainingRows, pruningRows, statescope, cancellationToken);
+    }
+
+    [StorableType("E84ACC40-5694-4E40-A947-190673643206")]
+    private sealed class ConfidenceRegressionNodeTreeModel : RegressionNodeTreeModel, IConfidenceRegressionModel {
+      #region HLConstructors & Cloning
+      [StorableConstructor]
+      private ConfidenceRegressionNodeTreeModel(StorableConstructorFlag _) : base(_) { }
+      private ConfidenceRegressionNodeTreeModel(ConfidenceRegressionNodeTreeModel original, Cloner cloner) : base(original, cloner) { }
+      public ConfidenceRegressionNodeTreeModel(string targetVariable) : base(targetVariable) { }
+      public override IDeepCloneable Clone(Cloner cloner) {
+        return new ConfidenceRegressionNodeTreeModel(this, cloner);
+      }
+      #endregion
+
+      public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
+        if (Root == null) throw new NotSupportedException("The model has not been built yet");
+        return ((IConfidenceRegressionModel)Root).GetEstimatedVariances(dataset, rows);
+      }
+      public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+        return new ConfidenceRegressionSolution(this, problemData);
+      }
+    }
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/MetaModels/RegressionRuleModel.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/MetaModels/RegressionRuleModel.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/MetaModels/RegressionRuleModel.cs	(revision 17078)
@@ -0,0 +1,179 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("425AF262-A756-4E9A-B76F-4D2480BEA4FD")]
+  public class RegressionRuleModel : RegressionModel, IM5Model {
+    #region Properties
+    [Storable]
+    public string[] SplitAttributes { get; set; }
+    [Storable]
+    private double[] SplitValues { get; set; }
+    [Storable]
+    private Comparison[] Comparisons { get; set; }
+    [Storable]
+    private IRegressionModel RuleModel { get; set; }
+    [Storable]
+    private IReadOnlyList<string> variables;
+    #endregion
+
+    #region HLConstructors
+    [StorableConstructor]
+    protected RegressionRuleModel(StorableConstructorFlag _) : base(_) { }
+    protected RegressionRuleModel(RegressionRuleModel original, Cloner cloner) : base(original, cloner) {
+      if (original.SplitAttributes != null) SplitAttributes = original.SplitAttributes.ToArray();
+      if (original.SplitValues != null) SplitValues = original.SplitValues.ToArray();
+      if (original.Comparisons != null) Comparisons = original.Comparisons.ToArray();
+      RuleModel = cloner.Clone(original.RuleModel);
+      if (original.variables != null) variables = original.variables.ToList();
+    }
+    private RegressionRuleModel(string target) : base(target) { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RegressionRuleModel(this, cloner);
+    }
+    #endregion
+
+    internal static RegressionRuleModel CreateRuleModel(string target, RegressionTreeParameters regressionTreeParams) {
+      return regressionTreeParams.LeafModel.ProvidesConfidence ? new ConfidenceRegressionRuleModel(target) : new RegressionRuleModel(target);
+    }
+
+    #region IRegressionModel
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return variables; }
+    }
+
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      if (RuleModel == null) throw new NotSupportedException("The model has not been built correctly");
+      return RuleModel.GetEstimatedValues(dataset, rows);
+    }
+
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RegressionSolution(this, problemData);
+    }
+    #endregion
+
+    public void Build(IReadOnlyList<int> trainingRows, IReadOnlyList<int> pruningRows, IScope statescope, ResultCollection results, CancellationToken cancellationToken) {
+      var regressionTreeParams = (RegressionTreeParameters)statescope.Variables[M5Regression.RegressionTreeParameterVariableName].Value;
+      variables = regressionTreeParams.AllowedInputVariables.ToList();
+
+      //build tree and select node with maximum coverage
+      var tree = RegressionNodeTreeModel.CreateTreeModel(regressionTreeParams.TargetVariable, regressionTreeParams);
+      tree.BuildModel(trainingRows, pruningRows, statescope, results, cancellationToken);
+      var nodeModel = tree.Root.EnumerateNodes().Where(x => x.IsLeaf).MaxItems(x => x.NumSamples).First();
+
+      var satts = new List<string>();
+      var svals = new List<double>();
+      var reops = new List<Comparison>();
+
+      //extract splits
+      for (var temp = nodeModel; temp.Parent != null; temp = temp.Parent) {
+        satts.Add(temp.Parent.SplitAttribute);
+        svals.Add(temp.Parent.SplitValue);
+        reops.Add(temp.Parent.Left == temp ? Comparison.LessEqual : Comparison.Greater);
+      }
+      Comparisons = reops.ToArray();
+      SplitAttributes = satts.ToArray();
+      SplitValues = svals.ToArray();
+      int np;
+      RuleModel = regressionTreeParams.LeafModel.BuildModel(trainingRows.Union(pruningRows).Where(r => Covers(regressionTreeParams.Data, r)).ToArray(), regressionTreeParams, cancellationToken, out np);
+    }
+
+    public void Update(IReadOnlyList<int> rows, IScope statescope, CancellationToken cancellationToken) {
+      var regressionTreeParams = (RegressionTreeParameters)statescope.Variables[M5Regression.RegressionTreeParameterVariableName].Value;
+      int np;
+      RuleModel = regressionTreeParams.LeafModel.BuildModel(rows, regressionTreeParams, cancellationToken, out np);
+    }
+
+    public bool Covers(IDataset dataset, int row) {
+      return !SplitAttributes.Where((t, i) => !Comparisons[i].Compare(dataset.GetDoubleValue(t, row), SplitValues[i])).Any();
+    }
+
+    public string ToCompactString() {
+      var mins = new Dictionary<string, double>();
+      var maxs = new Dictionary<string, double>();
+      for (var i = 0; i < SplitAttributes.Length; i++) {
+        var n = SplitAttributes[i];
+        var v = SplitValues[i];
+        if (!mins.ContainsKey(n)) mins.Add(n, double.NegativeInfinity);
+        if (!maxs.ContainsKey(n)) maxs.Add(n, double.PositiveInfinity);
+        if (Comparisons[i] == Comparison.LessEqual) maxs[n] = Math.Min(maxs[n], v);
+        else mins[n] = Math.Max(mins[n], v);
+      }
+      if (maxs.Count == 0) return "";
+      var s = new StringBuilder();
+      foreach (var key in maxs.Keys)
+        s.Append(string.Format("{0} ∈ [{1:e2}; {2:e2}] && ", key, mins[key], maxs[key]));
+      s.Remove(s.Length - 4, 4);
+      return s.ToString();
+    }
+
+    [StorableType("7302AA30-9F58-42F3-BF6A-ECF1536508AB")]
+    private sealed class ConfidenceRegressionRuleModel : RegressionRuleModel, IConfidenceRegressionModel {
+      #region HLConstructors
+      [StorableConstructor]
+      private ConfidenceRegressionRuleModel(StorableConstructorFlag _) : base(_) { }
+      private ConfidenceRegressionRuleModel(ConfidenceRegressionRuleModel original, Cloner cloner) : base(original, cloner) { }
+      public ConfidenceRegressionRuleModel(string targetAttr) : base(targetAttr) { }
+      public override IDeepCloneable Clone(Cloner cloner) {
+        return new ConfidenceRegressionRuleModel(this, cloner);
+      }
+      #endregion
+
+      public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
+        return ((IConfidenceRegressionModel)RuleModel).GetEstimatedVariances(dataset, rows);
+      }
+
+      public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+        return new ConfidenceRegressionSolution(this, problemData);
+      }
+    }
+  }
+
+  [StorableType("152DECE4-2692-4D53-B290-974806ADCD72")]
+  internal enum Comparison {
+    LessEqual,
+    Greater
+  }
+
+  internal static class ComparisonExtentions {
+    public static bool Compare(this Comparison op, double x, double y) {
+      switch (op) {
+        case Comparison.Greater:
+          return x > y;
+        case Comparison.LessEqual:
+          return x <= y;
+        default:
+          throw new ArgumentOutOfRangeException(op.ToString(), op, null);
+      }
+    }
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/MetaModels/RegressionRuleSetModel.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/MetaModels/RegressionRuleSetModel.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/MetaModels/RegressionRuleSetModel.cs	(revision 17078)
@@ -0,0 +1,186 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("7B4D9AE9-0456-4029-80A6-CCB5E33CE356")]
+  public class RegressionRuleSetModel : RegressionModel, IM5Model {
+    private const string NumRulesResultName = "Number of rules";
+    private const string CoveredInstancesResultName = "Covered instances";
+    public const string RuleSetStateVariableName = "RuleSetState";
+
+    #region Properties
+    [Storable]
+    internal List<RegressionRuleModel> Rules { get; private set; }
+    #endregion
+
+    #region HLConstructors & Cloning
+    [StorableConstructor]
+    protected RegressionRuleSetModel(StorableConstructorFlag _) : base(_) { }
+    protected RegressionRuleSetModel(RegressionRuleSetModel original, Cloner cloner) : base(original, cloner) {
+      if (original.Rules != null) Rules = original.Rules.Select(cloner.Clone).ToList();
+    }
+    protected RegressionRuleSetModel(string targetVariable) : base(targetVariable) { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RegressionRuleSetModel(this, cloner);
+    }
+    #endregion
+
+    internal static RegressionRuleSetModel CreateRuleModel(string targetAttr, RegressionTreeParameters regressionTreeParams) {
+      return regressionTreeParams.LeafModel.ProvidesConfidence ? new ConfidenceRegressionRuleSetModel(targetAttr) : new RegressionRuleSetModel(targetAttr);
+    }
+
+    #region RegressionModel
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get {
+        var f = Rules.FirstOrDefault();
+        return f != null ? (f.VariablesUsedForPrediction ?? new List<string>()) : new List<string>();
+      }
+    }
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      if (Rules == null) throw new NotSupportedException("The model has not been built yet");
+      return rows.Select(row => GetEstimatedValue(dataset, row));
+    }
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RegressionSolution(this, problemData);
+    }
+    #endregion
+
+    #region IM5Model
+    public void Build(IReadOnlyList<int> trainingRows, IReadOnlyList<int> pruningRows, IScope stateScope, ResultCollection results, CancellationToken cancellationToken) {
+      var regressionTreeParams = (RegressionTreeParameters)stateScope.Variables[M5Regression.RegressionTreeParameterVariableName].Value;
+      var ruleSetState = (RuleSetState)stateScope.Variables[RuleSetStateVariableName].Value;
+
+      if (ruleSetState.Code <= 0) {
+        ruleSetState.Rules.Clear();
+        ruleSetState.TrainingRows = trainingRows;
+        ruleSetState.PruningRows = pruningRows;
+        ruleSetState.Code = 1;
+      }
+
+      do {
+        var tempRule = RegressionRuleModel.CreateRuleModel(regressionTreeParams.TargetVariable, regressionTreeParams);
+        cancellationToken.ThrowIfCancellationRequested();
+
+        if (!results.ContainsKey(NumRulesResultName)) results.Add(new Result(NumRulesResultName, new IntValue(0)));
+        if (!results.ContainsKey(CoveredInstancesResultName)) results.Add(new Result(CoveredInstancesResultName, new IntValue(0)));
+
+        var t1 = ruleSetState.TrainingRows.Count;
+        tempRule.Build(ruleSetState.TrainingRows, ruleSetState.PruningRows, stateScope, results, cancellationToken);
+        ruleSetState.TrainingRows = ruleSetState.TrainingRows.Where(i => !tempRule.Covers(regressionTreeParams.Data, i)).ToArray();
+        ruleSetState.PruningRows = ruleSetState.PruningRows.Where(i => !tempRule.Covers(regressionTreeParams.Data, i)).ToArray();
+        ruleSetState.Rules.Add(tempRule);
+        ((IntValue)results[NumRulesResultName].Value).Value++;
+        ((IntValue)results[CoveredInstancesResultName].Value).Value += t1 - ruleSetState.TrainingRows.Count;
+      }
+      while (ruleSetState.TrainingRows.Count > 0);
+      Rules = ruleSetState.Rules;
+    }
+    public void Update(IReadOnlyList<int> rows, IScope stateScope, CancellationToken cancellationToken) {
+      foreach (var rule in Rules) rule.Update(rows, stateScope, cancellationToken);
+    }
+    public static void Initialize(IScope stateScope) {
+      stateScope.Variables.Add(new Variable(RuleSetStateVariableName, new RuleSetState()));
+    }
+    #endregion
+
+    #region Helpers
+    private double GetEstimatedValue(IDataset dataset, int row) {
+      foreach (var rule in Rules) {
+        if (rule.Covers(dataset, row))
+          return rule.GetEstimatedValues(dataset, row.ToEnumerable()).Single();
+      }
+      throw new ArgumentException("Instance is not covered by any rule");
+    }
+    #endregion
+
+    [StorableType("E114F3C9-3C1F-443D-8270-0E10CE12F2A0")]
+    public class RuleSetState : Item {
+      [Storable]
+      public List<RegressionRuleModel> Rules = new List<RegressionRuleModel>();
+      [Storable]
+      public IReadOnlyList<int> TrainingRows = new List<int>();
+      [Storable]
+      public IReadOnlyList<int> PruningRows = new List<int>();
+
+      //State.Code values denote the current action (for pausing)
+      //0...nothing has been done;
+      //1...splitting nodes;
+      [Storable]
+      public int Code = 0;
+
+      #region HLConstructors & Cloning
+      [StorableConstructor]
+      protected RuleSetState(StorableConstructorFlag _) : base(_) { }
+      protected RuleSetState(RuleSetState original, Cloner cloner) : base(original, cloner) {
+        Rules = original.Rules.Select(cloner.Clone).ToList();
+        TrainingRows = original.TrainingRows.ToList();
+        PruningRows = original.PruningRows.ToList();
+
+        Code = original.Code;
+      }
+      public RuleSetState() { }
+      public override IDeepCloneable Clone(Cloner cloner) {
+        return new RuleSetState(this, cloner);
+      }
+      #endregion
+    }
+
+    [StorableType("52E7992B-94CC-4960-AA82-1A399BE735C6")]
+    private sealed class ConfidenceRegressionRuleSetModel : RegressionRuleSetModel, IConfidenceRegressionModel {
+      #region HLConstructors & Cloning
+      [StorableConstructor]
+      private ConfidenceRegressionRuleSetModel(StorableConstructorFlag _) : base(_) { }
+      private ConfidenceRegressionRuleSetModel(ConfidenceRegressionRuleSetModel original, Cloner cloner) : base(original, cloner) { }
+      public ConfidenceRegressionRuleSetModel(string targetVariable) : base(targetVariable) { }
+      public override IDeepCloneable Clone(Cloner cloner) {
+        return new ConfidenceRegressionRuleSetModel(this, cloner);
+      }
+      #endregion
+
+      #region IConfidenceRegressionModel
+      public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
+        if (Rules == null) throw new NotSupportedException("The model has not been built yet");
+        return rows.Select(row => GetEstimatedVariance(dataset, row));
+      }
+      public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+        return new ConfidenceRegressionSolution(this, problemData);
+      }
+      private double GetEstimatedVariance(IDataset dataset, int row) {
+        foreach (var rule in Rules) {
+          if (rule.Covers(dataset, row)) return ((IConfidenceRegressionModel)rule).GetEstimatedVariances(dataset, row.ToEnumerable()).Single();
+        }
+        throw new ArgumentException("Instance is not covered by any rule");
+      }
+      #endregion
+    }
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Plugin.cs.frame
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Plugin.cs.frame	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Plugin.cs.frame	(revision 17078)
@@ -0,0 +1,48 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.PluginInfrastructure;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Plugin class for HeuristicLab.Algorithms.DataAnalysis plugin.
+  /// </summary>
+  [Plugin("HeuristicLab.Algorithms.DataAnalysis.M5", "Provides the M5' regression algorithm", "3.4.0.$WCREV$")]
+  [PluginFile("HeuristicLab.Algorithms.DataAnalysis.M5-3.4.dll", PluginFileType.Assembly)]
+  [PluginDependency("HeuristicLab.ALGLIB", "3.7")]
+  [PluginDependency("HeuristicLab.Attic", "1.0")]
+  [PluginDependency("HeuristicLab.Algorithms.DataAnalysis", "3.4")]
+  [PluginDependency("HeuristicLab.Algorithms.DataAnalysis.Glmnet", "3.4")]
+  [PluginDependency("HeuristicLab.Analysis", "3.3")]
+  [PluginDependency("HeuristicLab.Collections", "3.3")]
+  [PluginDependency("HeuristicLab.Common", "3.3")]
+  [PluginDependency("HeuristicLab.Core", "3.3")]
+  [PluginDependency("HeuristicLab.Data", "3.3")]
+  [PluginDependency("HeuristicLab.Encodings.PermutationEncoding", "3.3")]
+  [PluginDependency("HeuristicLab.Encodings.SymbolicExpressionTreeEncoding", "3.4")]
+  [PluginDependency("HeuristicLab.Optimization", "3.3")]
+  [PluginDependency("HeuristicLab.Parameters", "3.3")]
+  [PluginDependency("HeuristicLab.Problems.DataAnalysis", "3.4")]
+  [PluginDependency("HeuristicLab.Random", "3.3")]
+
+  public class Plugin : PluginBase {   
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Properties/AssemblyInfo.cs.frame
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Properties/AssemblyInfo.cs.frame	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Properties/AssemblyInfo.cs.frame	(revision 17078)
@@ -0,0 +1,55 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using HeuristicLab.PluginInfrastructure;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("HeuristicLab.Algorithms.DataAnalysis.M5")]
+[assembly: AssemblyDescription("Provides the M5' regression algorithm")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("HeuristicLab")]
+[assembly: AssemblyCopyright("(c) 2002-2019 HEAL")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("3.4.0")]
+[assembly: AssemblyFileVersion("3.4.0.$WCREV$")]
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Pruning/ComplexityPruning.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Pruning/ComplexityPruning.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Pruning/ComplexityPruning.cs	(revision 17078)
@@ -0,0 +1,325 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("B643D965-D13F-415D-8589-3F3527460347")]
+  public class ComplexityPruning : ParameterizedNamedItem, IPruning {
+    public const string PruningStateVariableName = "PruningState";
+
+    public const string PruningStrengthParameterName = "PruningStrength";
+    public const string PruningDecayParameterName = "PruningDecay";
+    public const string FastPruningParameterName = "FastPruning";
+
+    public IFixedValueParameter<DoubleValue> PruningStrengthParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[PruningStrengthParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> PruningDecayParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[PruningDecayParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> FastPruningParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[FastPruningParameterName]; }
+    }
+
+    public double PruningStrength {
+      get { return PruningStrengthParameter.Value.Value; }
+      set { PruningStrengthParameter.Value.Value = value; }
+    }
+    public double PruningDecay {
+      get { return PruningDecayParameter.Value.Value; }
+      set { PruningDecayParameter.Value.Value = value; }
+    }
+    public bool FastPruning {
+      get { return FastPruningParameter.Value.Value; }
+      set { FastPruningParameter.Value.Value = value; }
+    }
+
+    #region Constructors & Cloning
+    [StorableConstructor]
+    protected ComplexityPruning(StorableConstructorFlag _) : base(_) { }
+    protected ComplexityPruning(ComplexityPruning original, Cloner cloner) : base(original, cloner) { }
+    public ComplexityPruning() {
+      Parameters.Add(new FixedValueParameter<DoubleValue>(PruningStrengthParameterName, "The strength of the pruning. Higher values force the algorithm to create simpler models (default=2.0).", new DoubleValue(2.0)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(PruningDecayParameterName, "Pruning decay allows nodes higher up in the tree to be more stable (default=1.0).", new DoubleValue(1.0)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(FastPruningParameterName, "Accelerate pruning by using linear models instead of leaf models (default=true).", new BoolValue(true)));
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new ComplexityPruning(this, cloner);
+    }
+    #endregion
+
+    #region IPruning
+    public int MinLeafSize(IRegressionProblemData pd, ILeafModel leafModel) {
+      return (FastPruning ? new LinearLeaf() : leafModel).MinLeafSize(pd);
+    }
+    public void Initialize(IScope states) {
+      states.Variables.Add(new Variable(PruningStateVariableName, new PruningState()));
+    }
+
+    public void Prune(RegressionNodeTreeModel treeModel, IReadOnlyList<int> trainingRows, IReadOnlyList<int> pruningRows, IScope statescope, CancellationToken cancellationToken) {
+      var regressionTreeParams = (RegressionTreeParameters)statescope.Variables[M5Regression.RegressionTreeParameterVariableName].Value;
+      var state = (PruningState)statescope.Variables[PruningStateVariableName].Value;
+
+      var leaf = FastPruning ? new LinearLeaf() : regressionTreeParams.LeafModel;
+      if (state.Code <= 1) {
+        InstallModels(treeModel, state, trainingRows, pruningRows, leaf, regressionTreeParams, cancellationToken);
+        cancellationToken.ThrowIfCancellationRequested();
+      }
+      if (state.Code <= 2) {
+        AssignPruningThresholds(treeModel, state, PruningDecay);
+        cancellationToken.ThrowIfCancellationRequested();
+      }
+      if (state.Code <= 3) {
+        UpdateThreshold(treeModel, state);
+        cancellationToken.ThrowIfCancellationRequested();
+      }
+      if (state.Code <= 4) {
+        Prune(treeModel, state, PruningStrength);
+        cancellationToken.ThrowIfCancellationRequested();
+      }
+
+      state.Code = 5;
+    }
+    #endregion
+
+    private static void InstallModels(RegressionNodeTreeModel tree, PruningState state, IReadOnlyList<int> trainingRows, IReadOnlyList<int> pruningRows, ILeafModel leaf, RegressionTreeParameters regressionTreeParams, CancellationToken cancellationToken) {
+      if (state.Code == 0) {
+        state.FillBottomUp(tree, trainingRows, pruningRows, regressionTreeParams.Data);
+        state.Code = 1;
+      }
+      while (state.nodeQueue.Count != 0) {
+        cancellationToken.ThrowIfCancellationRequested();
+        var n = state.nodeQueue.Peek();
+        var training = state.trainingRowsQueue.Peek();
+        var pruning = state.pruningRowsQueue.Peek();
+        BuildPruningModel(n, leaf, training, pruning, state, regressionTreeParams, cancellationToken);
+        state.nodeQueue.Dequeue();
+        state.trainingRowsQueue.Dequeue();
+        state.pruningRowsQueue.Dequeue();
+      }
+    }
+
+    private static void AssignPruningThresholds(RegressionNodeTreeModel tree, PruningState state, double pruningDecay) {
+      if (state.Code == 1) {
+        state.FillBottomUp(tree);
+        state.Code = 2;
+      }
+      while (state.nodeQueue.Count != 0) {
+        var n = state.nodeQueue.Dequeue();
+        if (n.IsLeaf) continue;
+        n.PruningStrength = PruningThreshold(state.pruningSizes[n], state.modelComplexities[n], state.nodeComplexities[n], state.modelErrors[n], SubtreeError(n, state.pruningSizes, state.modelErrors), pruningDecay);
+      }
+    }
+
+    private static void UpdateThreshold(RegressionNodeTreeModel tree, PruningState state) {
+      if (state.Code == 2) {
+        state.FillTopDown(tree);
+        state.Code = 3;
+      }
+      while (state.nodeQueue.Count != 0) {
+        var n = state.nodeQueue.Dequeue();
+        if (n.IsLeaf || n.Parent == null || double.IsNaN(n.Parent.PruningStrength)) continue;
+        n.PruningStrength = Math.Min(n.PruningStrength, n.Parent.PruningStrength);
+      }
+    }
+
+    private static void Prune(RegressionNodeTreeModel tree, PruningState state, double pruningStrength) {
+      if (state.Code == 3) {
+        state.FillTopDown(tree);
+        state.Code = 4;
+      }
+      while (state.nodeQueue.Count != 0) {
+        var n = state.nodeQueue.Dequeue();
+        if (n.IsLeaf || pruningStrength <= n.PruningStrength) continue;
+        n.ToLeaf();
+      }
+    }
+
+    private static void BuildPruningModel(RegressionNodeModel regressionNode, ILeafModel leaf, IReadOnlyList<int> trainingRows, IReadOnlyList<int> pruningRows, PruningState state, RegressionTreeParameters regressionTreeParams, CancellationToken cancellationToken) {
+      //create regressionProblemdata from pruning data 
+      var vars = regressionTreeParams.AllowedInputVariables.Concat(new[] { regressionTreeParams.TargetVariable }).ToArray();
+      var reducedData = new Dataset(vars, vars.Select(x => regressionTreeParams.Data.GetDoubleValues(x, pruningRows).ToList()));
+      var pd = new RegressionProblemData(reducedData, regressionTreeParams.AllowedInputVariables, regressionTreeParams.TargetVariable);
+      pd.TrainingPartition.Start = pd.TrainingPartition.End = pd.TestPartition.Start = 0;
+      pd.TestPartition.End = reducedData.Rows;
+
+      //build pruning model
+      int numModelParams;
+      var model = leaf.BuildModel(trainingRows, regressionTreeParams, cancellationToken, out numModelParams);
+
+      //record error and complexities
+      var rmsModel = model.CreateRegressionSolution(pd).TestRootMeanSquaredError;
+      state.pruningSizes.Add(regressionNode, pruningRows.Count);
+      state.modelErrors.Add(regressionNode, rmsModel);
+      state.modelComplexities.Add(regressionNode, numModelParams);
+      if (regressionNode.IsLeaf) { state.nodeComplexities[regressionNode] = state.modelComplexities[regressionNode]; } else { state.nodeComplexities.Add(regressionNode, state.nodeComplexities[regressionNode.Left] + state.nodeComplexities[regressionNode.Right] + 1); }
+    }
+
+    private static double PruningThreshold(double noIntances, double modelParams, double nodeParams, double modelError, double nodeError, double w) {
+      var res = modelError / nodeError;
+      if (modelError.IsAlmost(nodeError)) res = 1.0;
+      res /= Math.Pow((nodeParams + noIntances) / (2 * (modelParams + noIntances)), w);
+      return res;
+    }
+
+    private static double SubtreeError(RegressionNodeModel regressionNode, IDictionary<RegressionNodeModel, int> pruningSizes,
+      IDictionary<RegressionNodeModel, double> modelErrors) {
+      if (regressionNode.IsLeaf) return modelErrors[regressionNode];
+      var errorL = SubtreeError(regressionNode.Left, pruningSizes, modelErrors);
+      var errorR = SubtreeError(regressionNode.Right, pruningSizes, modelErrors);
+      errorL = errorL * errorL * pruningSizes[regressionNode.Left];
+      errorR = errorR * errorR * pruningSizes[regressionNode.Right];
+      return Math.Sqrt((errorR + errorL) / pruningSizes[regressionNode]);
+    }
+
+    [StorableType("EAD60C7E-2C58-45C4-9697-6F735F518CFD")]
+    public class PruningState : Item {
+      [Storable]
+      public IDictionary<RegressionNodeModel, int> modelComplexities = new Dictionary<RegressionNodeModel, int>();
+      [Storable]
+      public IDictionary<RegressionNodeModel, int> nodeComplexities = new Dictionary<RegressionNodeModel, int>();
+      [Storable]
+      public IDictionary<RegressionNodeModel, int> pruningSizes = new Dictionary<RegressionNodeModel, int>();
+      [Storable]
+      public IDictionary<RegressionNodeModel, double> modelErrors = new Dictionary<RegressionNodeModel, double>();
+
+      [Storable]
+      public Queue<RegressionNodeModel> nodeQueue = new Queue<RegressionNodeModel>();
+      [Storable]
+      public Queue<IReadOnlyList<int>> trainingRowsQueue = new Queue<IReadOnlyList<int>>();
+      [Storable]
+      public Queue<IReadOnlyList<int>> pruningRowsQueue = new Queue<IReadOnlyList<int>>();
+
+      //State.Code values denote the current action (for pausing)
+      //0...nothing has been done;
+      //1...building Models;
+      //2...assigning threshold
+      //3...adjusting threshold
+      //4...pruning
+      //5...finished
+      [Storable]
+      public int Code = 0;
+
+      #region HLConstructors & Cloning
+      [StorableConstructor]
+      protected PruningState(StorableConstructorFlag _) : base(_) { }
+      protected PruningState(PruningState original, Cloner cloner) : base(original, cloner) {
+        modelComplexities = original.modelComplexities.ToDictionary(x => cloner.Clone(x.Key), x => x.Value);
+        nodeComplexities = original.nodeComplexities.ToDictionary(x => cloner.Clone(x.Key), x => x.Value);
+        pruningSizes = original.pruningSizes.ToDictionary(x => cloner.Clone(x.Key), x => x.Value);
+        modelErrors = original.modelErrors.ToDictionary(x => cloner.Clone(x.Key), x => x.Value);
+        nodeQueue = new Queue<RegressionNodeModel>(original.nodeQueue.Select(cloner.Clone));
+        trainingRowsQueue = new Queue<IReadOnlyList<int>>(original.trainingRowsQueue.Select(x => (IReadOnlyList<int>)x.ToArray()));
+        pruningRowsQueue = new Queue<IReadOnlyList<int>>(original.pruningRowsQueue.Select(x => (IReadOnlyList<int>)x.ToArray()));
+        Code = original.Code;
+      }
+      public PruningState() { }
+      public override IDeepCloneable Clone(Cloner cloner) {
+        return new PruningState(this, cloner);
+      }
+      #endregion
+
+      public void FillTopDown(RegressionNodeTreeModel tree) {
+        var helperQueue = new Queue<RegressionNodeModel>();
+        nodeQueue.Clear();
+
+        helperQueue.Enqueue(tree.Root);
+        nodeQueue.Enqueue(tree.Root);
+
+        while (helperQueue.Count != 0) {
+          var n = helperQueue.Dequeue();
+          if (n.IsLeaf) continue;
+          helperQueue.Enqueue(n.Left);
+          helperQueue.Enqueue(n.Right);
+          nodeQueue.Enqueue(n.Left);
+          nodeQueue.Enqueue(n.Right);
+        }
+      }
+
+      public void FillTopDown(RegressionNodeTreeModel tree, IReadOnlyList<int> pruningRows, IReadOnlyList<int> trainingRows, IDataset data) {
+        var helperQueue = new Queue<RegressionNodeModel>();
+        var trainingHelperQueue = new Queue<IReadOnlyList<int>>();
+        var pruningHelperQueue = new Queue<IReadOnlyList<int>>();
+        nodeQueue.Clear();
+        trainingRowsQueue.Clear();
+        pruningRowsQueue.Clear();
+
+        helperQueue.Enqueue(tree.Root);
+
+        trainingHelperQueue.Enqueue(trainingRows);
+        pruningHelperQueue.Enqueue(pruningRows);
+
+        nodeQueue.Enqueue(tree.Root);
+        trainingRowsQueue.Enqueue(trainingRows);
+        pruningRowsQueue.Enqueue(pruningRows);
+
+
+        while (helperQueue.Count != 0) {
+          var n = helperQueue.Dequeue();
+          var p = pruningHelperQueue.Dequeue();
+          var t = trainingHelperQueue.Dequeue();
+          if (n.IsLeaf) continue;
+
+          IReadOnlyList<int> leftPruning, rightPruning;
+          RegressionTreeUtilities.SplitRows(p, data, n.SplitAttribute, n.SplitValue, out leftPruning, out rightPruning);
+          IReadOnlyList<int> leftTraining, rightTraining;
+          RegressionTreeUtilities.SplitRows(t, data, n.SplitAttribute, n.SplitValue, out leftTraining, out rightTraining);
+
+          helperQueue.Enqueue(n.Left);
+          helperQueue.Enqueue(n.Right);
+          trainingHelperQueue.Enqueue(leftTraining);
+          trainingHelperQueue.Enqueue(rightTraining);
+          pruningHelperQueue.Enqueue(leftPruning);
+          pruningHelperQueue.Enqueue(rightPruning);
+
+          nodeQueue.Enqueue(n.Left);
+          nodeQueue.Enqueue(n.Right);
+          trainingRowsQueue.Enqueue(leftTraining);
+          trainingRowsQueue.Enqueue(rightTraining);
+          pruningRowsQueue.Enqueue(leftPruning);
+          pruningRowsQueue.Enqueue(rightPruning);
+        }
+      }
+
+      public void FillBottomUp(RegressionNodeTreeModel tree) {
+        FillTopDown(tree);
+        nodeQueue = new Queue<RegressionNodeModel>(nodeQueue.Reverse());
+      }
+
+      public void FillBottomUp(RegressionNodeTreeModel tree, IReadOnlyList<int> pruningRows, IReadOnlyList<int> trainingRows, IDataset data) {
+        FillTopDown(tree, pruningRows, trainingRows, data);
+        nodeQueue = new Queue<RegressionNodeModel>(nodeQueue.Reverse());
+        pruningRowsQueue = new Queue<IReadOnlyList<int>>(pruningRowsQueue.Reverse());
+        trainingRowsQueue = new Queue<IReadOnlyList<int>>(trainingRowsQueue.Reverse());
+      }
+    }
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Pruning/NoPruning.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Pruning/NoPruning.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Pruning/NoPruning.cs	(revision 17078)
@@ -0,0 +1,51 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("D67152AA-3533-45D2-B77B-4A0742FB4B92")]
+  [Item("NoPruning", "No pruning")]
+  public sealed class NoPruning : ParameterizedNamedItem, IPruning {
+    #region Constructors & Cloning
+    [StorableConstructor]
+    private NoPruning(StorableConstructorFlag _) : base(_) { }
+    private NoPruning(NoPruning original, Cloner cloner) : base(original, cloner) { }
+    public NoPruning() { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NoPruning(this, cloner);
+    }
+    #endregion
+
+
+    public int MinLeafSize(IRegressionProblemData pd, ILeafModel leafModel) {
+      return 0;
+    }
+    public void Initialize(IScope states) { }
+
+    public void Prune(RegressionNodeTreeModel treeModel, IReadOnlyList<int> trainingRows, IReadOnlyList<int> pruningRows, IScope scope, CancellationToken cancellationToken) { }
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/CorrelationImpurityCalculator.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/CorrelationImpurityCalculator.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/CorrelationImpurityCalculator.cs	(revision 17078)
@@ -0,0 +1,88 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Helper class for incremental split calculation.
+  /// Used while moving a potential splitter along the ordered training instances
+  /// </summary>
+  internal class CorreleationImpurityCalculator {
+    #region state
+    //Data
+    private readonly List<double> attributeValues;
+    private readonly List<double> targetValues;
+    private readonly double order;
+    private readonly UnivariateOnlineLR left;
+    private readonly UnivariateOnlineLR right;
+    #endregion
+
+    #region Properties
+    public double Impurity { get; private set; }
+    public double SplitValue {
+      get {
+        if (left.Size <= 0) return double.NegativeInfinity;
+        if (left.Size >= attributeValues.Count) return double.PositiveInfinity;
+        return (attributeValues[left.Size - 1] + attributeValues[left.Size]) / 2;
+      }
+    }
+    public bool ValidPosition {
+      get { return !attributeValues[left.Size - 1].IsAlmost(attributeValues[left.Size]); }
+    }
+    public int LeftSize {
+      get { return left.Size; }
+    }
+    #endregion
+
+    #region Constructors
+    public CorreleationImpurityCalculator(int partition, IEnumerable<double> atts, IEnumerable<double> targets, double order) {
+      if (order <= 0) throw new ArgumentException("Splitter order must be larger than 0");
+      this.order = order;
+      attributeValues = atts.ToList();
+      targetValues = targets.ToList();
+      left = new UnivariateOnlineLR(attributeValues.Take(partition).ToList(), targetValues.Take(partition).ToList());
+      right = new UnivariateOnlineLR(attributeValues.Skip(partition).ToList(), targetValues.Skip(partition).ToList());
+      UpdateImpurity();
+    }
+    #endregion
+
+    #region IImpurityCalculator
+    public void Increment() {
+      var target = targetValues[left.Size];
+      var att = attributeValues[left.Size];
+      left.Add(att, target);
+      right.Remove(att, target);
+      UpdateImpurity();
+    }
+    #endregion
+
+    private void UpdateImpurity() {
+      var yl = Math.Pow(left.Ssr, 1.0 / order);
+      var yr = Math.Pow(right.Ssr, 1.0 / order);
+      if (left.Size > 1 && right.Size > 1) Impurity = -yl - yr;
+      else Impurity = double.MinValue;
+    }
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/CorrelationSplitter.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/CorrelationSplitter.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/CorrelationSplitter.cs	(revision 17078)
@@ -0,0 +1,98 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("EC3A5009-EE84-4E1A-A537-20F6F1224842")]
+  [Item("CorrelationSplitter", "An experimental split selector that uses correlation coefficients")]
+  public sealed class CorrelationSplitter : SplitterBase {
+    public const string OrderParameterName = "Order";
+    public IFixedValueParameter<DoubleValue> OrderParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[OrderParameterName]; }
+    }
+    public double Order {
+      get { return OrderParameter.Value.Value; }
+      set { OrderParameter.Value.Value = value; }
+    }
+
+    #region Constructors & Cloning
+    [StorableConstructor]
+    private CorrelationSplitter(StorableConstructorFlag _) { }
+    private CorrelationSplitter(CorrelationSplitter original, Cloner cloner) : base(original, cloner) { }
+    public CorrelationSplitter() {
+      Parameters.Add(new FixedValueParameter<DoubleValue>(OrderParameterName, "The exponent in the split calculation ssrLeft^(1/Order)+ssrRight^(1/Order) (default=1.0).", new DoubleValue(1)));
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CorrelationSplitter(this, cloner);
+    }
+    #endregion
+
+    #region ISplitType
+    protected override void AttributeSplit(IReadOnlyList<double> attValues, IReadOnlyList<double> targetValues, int minLeafSize, out int leftSize, out double maxImpurity, out double splitValue) {
+      leftSize = -1;
+      splitValue = double.MinValue;
+      maxImpurity = double.NegativeInfinity;
+      var splitValues = new List<double>();
+      var splitSizes = new List<int>();
+      var length = attValues.Count;
+
+      var start = minLeafSize;
+      while (start < length && attValues[start - 1].IsAlmost(attValues[start]))
+        start++;
+      if (start >= length) return;
+
+      var imp = new CorreleationImpurityCalculator(minLeafSize, attValues, targetValues, Order);
+      maxImpurity = imp.Impurity;
+      splitValues.Add(imp.SplitValue);
+      splitSizes.Add(imp.LeftSize);
+
+      while (imp.LeftSize < length - minLeafSize) {
+        imp.Increment();
+        if (!imp.ValidPosition) continue; //splits can not be made between to equal points
+
+        if (imp.Impurity.IsAlmost(maxImpurity)) {
+          splitValues.Add(imp.SplitValue);
+          splitSizes.Add(imp.LeftSize);
+          continue;
+        }
+
+        if (imp.Impurity < maxImpurity) continue;
+        splitValues.Clear();
+        splitSizes.Clear();
+        maxImpurity = imp.Impurity;
+        splitValues.Add(imp.SplitValue);
+        splitSizes.Add(imp.LeftSize);
+      }
+
+      var j = splitSizes.Count / 2;
+      if (splitSizes.Count == 0) return;
+      splitValue = splitValues[j];
+      leftSize = splitSizes[j];
+    }
+    #endregion
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/M5Splitter.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/M5Splitter.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/M5Splitter.cs	(revision 17078)
@@ -0,0 +1,80 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("502B1429-7A28-45C1-A60A-93E72CB3AF4A")]
+  [Item("M5Splitter", "A split selector that uses the ratio between Variances^(1/Order) to determine good splits.")]
+  public sealed class M5Splitter : SplitterBase {
+    public const string OrderParameterName = "Order";
+    public IFixedValueParameter<DoubleValue> OrderParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[OrderParameterName]; }
+    }
+    public double Order {
+      get { return OrderParameter.Value.Value; }
+      set { OrderParameter.Value.Value = value; }
+    }
+
+    #region Constructors & Cloning
+    [StorableConstructor]
+    private M5Splitter(StorableConstructorFlag _) { }
+    private M5Splitter(M5Splitter original, Cloner cloner) : base(original, cloner) { }
+    public M5Splitter() {
+      Parameters.Add(new FixedValueParameter<DoubleValue>(OrderParameterName, "The exponent in the split calculation sum (x_i - x_avg)^Order (default=5).", new DoubleValue(5)));
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new M5Splitter(this, cloner);
+    }
+    #endregion
+
+    protected override void AttributeSplit(IReadOnlyList<double> attValues, IReadOnlyList<double> targetValues, int minLeafSize, out int position, out double maxImpurity, out double splitValue) {
+      position = 0;
+      maxImpurity = double.NegativeInfinity;
+      splitValue = 0.0;
+      var length = targetValues.Count;
+
+      // weka code
+      var low = 0;
+      var high = length - 1;
+      if (high - low + 1 < 4) return;
+      var len = Math.Max(minLeafSize - 1, high - low + 1 < 5 ? 1 : (high - low + 1) / 5);
+      position = low;
+      var part = low + len - 1;
+      var imp = new OrderImpurityCalculator(part + 1, targetValues, Order);
+
+      for (var i = low + len; i < high - len; i++) {
+        imp.Increment(targetValues[i], OrderImpurityCalculator.IncrementType.Left);
+        if (attValues[i].IsAlmost(attValues[i + 1])) continue; //splits can not be made between to equal points
+        if (imp.Impurity < maxImpurity) continue;
+        maxImpurity = imp.Impurity;
+        splitValue = (attValues[i] + attValues[i + 1]) / 2;
+        position = i;
+      }
+    }
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/NeumaierSum.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/NeumaierSum.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/NeumaierSum.cs	(revision 17078)
@@ -0,0 +1,63 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Runtime.CompilerServices;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Helper class for incremental split calculation.
+  /// Used while moving a potential Splitter along the ordered training Instances
+  /// </summary>
+  internal class NeumaierSum {
+    #region state
+    private double sum;
+    private double correction;
+    #endregion
+
+    #region Constructors
+    public NeumaierSum(double startvalue) {
+      sum = startvalue;
+      correction = 0;
+    }
+    #endregion
+
+    [MethodImpl(MethodImplOptions.NoOptimization)]
+    public void Add(double value) {
+      var t = sum + value;
+      var absSum = sum > 0 ? sum : -sum;
+      var absv = value > 0 ? value : -value;
+      if (absSum >= absv)
+        correction += (sum - t) + value;
+      else
+        correction += (value - t) + sum;
+      sum = t;
+    }
+
+    public double Get() {
+      return sum + correction;
+    }
+
+    public void Mul(double value) {
+      sum *= value;
+      correction *= value;
+    }
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/OrderImpurityCalculator.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/OrderImpurityCalculator.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/OrderImpurityCalculator.cs	(revision 17078)
@@ -0,0 +1,123 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Helper class for incremental split calculation.
+  /// Used while moving a potential splitter along the ordered training instances
+  /// </summary>
+  internal class OrderImpurityCalculator {
+    internal enum IncrementType {
+      Left,
+      Right,
+      None
+    }
+
+    #region Properties
+    private double SqSumLeft { get; set; }
+    private double SqSumRight { get; set; }
+    private double VarLeft { get; set; }
+    private double VarRight { get; set; }
+    private double Order { get; set; }
+    private double VarTotal { get; set; }
+    private int NoInstances { get; set; }
+
+    private double NoLeft { get; set; }
+    private double NoRight { get; set; }
+    private double SumLeft { get; set; }
+    private double SumRight { get; set; }
+    public double Impurity { get; private set; }
+    #endregion
+
+    #region Constructors
+    public OrderImpurityCalculator(int partition, IReadOnlyCollection<double> data, double order) {
+      var values = data;
+      NoInstances = data.Count;
+      VarTotal = values.VariancePop();
+
+      values = data.Take(partition).ToArray();
+      NoLeft = partition;
+      SumLeft = values.Sum();
+      SqSumLeft = values.Sum(x => x * x);
+
+      values = data.Skip(partition).ToArray();
+      NoRight = NoInstances - NoLeft;
+      SumRight = values.Sum();
+      SqSumRight = values.Sum(x => x * x);
+
+      Order = order;
+      Increment(0.0, IncrementType.None);
+    }
+    #endregion
+
+    #region IImpurityCalculator
+    public void Increment(double value, IncrementType type) {
+      double y, yl, yr;
+      var valSq = value * value;
+
+      switch (type) {
+        case IncrementType.Left:
+          NoLeft++;
+          NoRight--;
+          SumLeft += value;
+          SumRight -= value;
+          SqSumLeft += valSq;
+          SqSumRight -= valSq;
+          break;
+        case IncrementType.Right:
+          NoLeft--;
+          NoRight++;
+          SumLeft -= value;
+          SumRight += value;
+          SqSumLeft -= valSq;
+          SqSumRight += valSq;
+          break;
+        case IncrementType.None:
+          break;
+        default:
+          throw new ArgumentOutOfRangeException(type.ToString(), type, null);
+      }
+
+      VarLeft = NoLeft <= 0 ? 0 : Math.Abs(NoLeft * SqSumLeft - SumLeft * SumLeft) / (NoLeft * NoLeft);
+      VarRight = NoRight <= 0 ? 0 : Math.Abs(NoRight * SqSumRight - SumRight * SumRight) / (NoRight * NoRight);
+
+      if (Order <= 0) throw new ArgumentException("Splitter order must be larger than 0");
+      if (Order.IsAlmost(1)) {
+        y = VarTotal;
+        yl = VarLeft;
+        yr = VarRight;
+      }
+      else {
+        y = Math.Pow(VarTotal, 1.0 / Order);
+        yl = Math.Pow(VarLeft, 1.0 / Order);
+        yr = Math.Pow(VarRight, 1.0 / Order);
+      }
+      if (NoLeft <= 0.0 || NoRight <= 0.0) Impurity = double.MinValue; //Splitter = 0;
+      else Impurity = y - (NoLeft * yl + NoRight * yr) / (NoRight + NoLeft);
+    }
+    #endregion
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/SplitterBase.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/SplitterBase.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/SplitterBase.cs	(revision 17078)
@@ -0,0 +1,140 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+using HEAL.Attic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableType("22DCCF28-8943-4622-BBD3-B2AB04F28C36")]
+  [Item("SplitterBase", "Abstract base class for splitters")]
+  public abstract class SplitterBase : ParameterizedNamedItem, ISplitter {
+    public const string SplittingStateVariableName = "SplittingState";
+
+    #region Constructors & Cloning
+    [StorableConstructor]
+    protected SplitterBase(StorableConstructorFlag _) { }
+    protected SplitterBase(SplitterBase original, Cloner cloner) : base(original, cloner) { }
+    public SplitterBase() { }
+    #endregion
+
+    #region ISplitType
+    public void Initialize(IScope states) {
+      states.Variables.Add(new Variable(SplittingStateVariableName, new SplittingState()));
+    }
+
+    public void Split(RegressionNodeTreeModel tree, IReadOnlyList<int> trainingRows, IScope stateScope, CancellationToken cancellationToken) {
+      var regressionTreeParams = (RegressionTreeParameters)stateScope.Variables[M5Regression.RegressionTreeParameterVariableName].Value;
+      var splittingState = (SplittingState)stateScope.Variables[SplittingStateVariableName].Value;
+      var variables = regressionTreeParams.AllowedInputVariables.ToArray();
+      var target = regressionTreeParams.TargetVariable;
+
+      if (splittingState.Code <= 0) {
+        splittingState.nodeQueue.Enqueue(tree.Root);
+        splittingState.trainingRowsQueue.Enqueue(trainingRows);
+        splittingState.Code = 1;
+      }
+      while (splittingState.nodeQueue.Count != 0) {
+        var n = splittingState.nodeQueue.Dequeue();
+        var rows = splittingState.trainingRowsQueue.Dequeue();
+
+        string attr;
+        double splitValue;
+        var isLeaf = !DecideSplit(new RegressionProblemData(RegressionTreeUtilities.ReduceDataset(regressionTreeParams.Data, rows, variables, target), variables, target), regressionTreeParams.MinLeafSize, out attr, out splitValue);
+        if (isLeaf) continue;
+
+        IReadOnlyList<int> leftRows, rightRows;
+        RegressionTreeUtilities.SplitRows(rows, regressionTreeParams.Data, attr, splitValue, out leftRows, out rightRows);
+        n.Split(regressionTreeParams, attr, splitValue, rows.Count);
+
+        splittingState.nodeQueue.Enqueue(n.Left);
+        splittingState.nodeQueue.Enqueue(n.Right);
+        splittingState.trainingRowsQueue.Enqueue(leftRows);
+        splittingState.trainingRowsQueue.Enqueue(rightRows);
+        cancellationToken.ThrowIfCancellationRequested();
+      }
+    }
+
+    protected virtual bool DecideSplit(IRegressionProblemData splitData, int minLeafSize, out string splitAttr, out double splitValue) {
+      var bestPos = -1;
+      var bestImpurity = double.MinValue;
+      var bestSplitValue = 0.0;
+      var bestSplitAttr = string.Empty;
+      splitAttr = bestSplitAttr;
+      splitValue = bestSplitValue;
+      if (splitData.Dataset.Rows < minLeafSize) return false;
+
+      // find best attribute for the splitter
+      foreach (var attr in splitData.AllowedInputVariables) {
+        int pos;
+        double impurity, sValue;
+        var sortedData = splitData.Dataset.GetDoubleValues(attr).Zip(splitData.Dataset.GetDoubleValues(splitData.TargetVariable), Tuple.Create).OrderBy(x => x.Item1).ToArray();
+        AttributeSplit(sortedData.Select(x => x.Item1).ToArray(), sortedData.Select(x => x.Item2).ToArray(), minLeafSize, out pos, out impurity, out sValue);
+        if (!(bestImpurity < impurity)) continue;
+        bestImpurity = impurity;
+        bestPos = pos;
+        bestSplitValue = sValue;
+        bestSplitAttr = attr;
+      }
+
+      splitAttr = bestSplitAttr;
+      splitValue = bestSplitValue;
+      //if no suitable split exists => leafNode
+      return bestPos + 1 >= minLeafSize && bestPos <= splitData.Dataset.Rows - minLeafSize;
+    }
+
+    protected abstract void AttributeSplit(IReadOnlyList<double> attValues, IReadOnlyList<double> targetValues, int minLeafSize, out int position, out double maxImpurity, out double splitValue);
+    #endregion
+
+    [StorableType("BC1149FD-370E-4F3A-92F5-6E519736D09A")]
+    public class SplittingState : Item {
+      [Storable]
+      public Queue<RegressionNodeModel> nodeQueue = new Queue<RegressionNodeModel>();
+      [Storable]
+      public Queue<IReadOnlyList<int>> trainingRowsQueue = new Queue<IReadOnlyList<int>>();
+
+      //State.Code values denote the current action (for pausing)
+      //0...nothing has been done;
+      //1...splitting nodes;
+      [Storable]
+      public int Code = 0;
+
+      #region HLConstructors & Cloning
+      [StorableConstructor]
+      protected SplittingState(StorableConstructorFlag _) : base(_) { }
+      protected SplittingState(SplittingState original, Cloner cloner) : base(original, cloner) {
+        nodeQueue = new Queue<RegressionNodeModel>(original.nodeQueue.Select(cloner.Clone));
+        trainingRowsQueue = new Queue<IReadOnlyList<int>>(original.trainingRowsQueue.Select(x => (IReadOnlyList<int>)x.ToArray()));
+        Code = original.Code;
+      }
+      public SplittingState() { }
+      public override IDeepCloneable Clone(Cloner cloner) {
+        return new SplittingState(this, cloner);
+      }
+      #endregion
+    }
+  }
+}
Index: /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/UnivariateOnlineLR.cs
===================================================================
--- /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/UnivariateOnlineLR.cs	(revision 17078)
+++ /trunk/HeuristicLab.Algorithms.DataAnalysis.DecisionTrees/3.4/Splitting/UnivariateOnlineLR.cs	(revision 17078)
@@ -0,0 +1,114 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Helper class for incremental split calculation.
+  /// Used while moving a potential splitter along the ordered training instances
+  /// </summary>
+  internal class UnivariateOnlineLR {
+    #region state
+    private readonly NeumaierSum targetMean;
+    private readonly NeumaierSum attributeMean;
+    private readonly NeumaierSum targetVarSum;
+    private readonly NeumaierSum attributeVarSum;
+    private readonly NeumaierSum comoment;
+    private readonly NeumaierSum ssr;
+    private int size;
+    #endregion
+
+    public double Ssr {
+      get { return ssr.Get(); }
+    }
+    public int Size {
+      get { return size; }
+    }
+
+    private double Beta {
+      get { return comoment.Get() / attributeVarSum.Get(); }
+    }
+    private double Alpha {
+      get { return targetMean.Get() - Beta * attributeMean.Get(); }
+    }
+
+    public UnivariateOnlineLR(ICollection<double> attributeValues, ICollection<double> targetValues) {
+      if (attributeValues.Count != targetValues.Count) throw new ArgumentException("Targets and Attributes need to have the same length");
+      size = attributeValues.Count;
+
+      var yMean = targetValues.Average();
+      var xMean = attributeValues.Average();
+      targetMean = new NeumaierSum(yMean);
+      attributeMean = new NeumaierSum(xMean);
+      targetVarSum = new NeumaierSum(targetValues.VariancePop() * size);
+      attributeVarSum = new NeumaierSum(attributeValues.VariancePop() * size);
+      comoment = new NeumaierSum(attributeValues.Zip(targetValues, (x, y) => (x - xMean) * (y - yMean)).Sum());
+
+      var beta = comoment.Get() / attributeVarSum.Get();
+      var alpha = yMean - beta * xMean;
+      ssr = new NeumaierSum(attributeValues.Zip(targetValues, (x, y) => y - alpha - beta * x).Sum(x => x * x));
+    }
+
+    public void Add(double attributeValue, double targetValue) {
+      var predictOld = Predict(attributeValue, targetValue);
+
+      size++;
+      var dx = attributeValue - attributeMean.Get();
+      var dy = targetValue - targetMean.Get();
+      attributeMean.Add(dx / size);
+      targetMean.Add(dy / size);
+      var dx2 = attributeValue - attributeMean.Get();
+      var dy2 = targetValue - targetMean.Get();
+      attributeVarSum.Add(dx * dx2);
+      targetVarSum.Add(dy * dy2);
+      comoment.Add(dx * dy2);
+
+      ssr.Add(predictOld * Predict(attributeValue, targetValue));
+    }
+
+    public void Remove(double attributeValue, double targetValue) {
+      var predictOld = Predict(attributeValue, targetValue);
+
+      var dx2 = attributeValue - attributeMean.Get();
+      var dy2 = targetValue - targetMean.Get();
+      attributeMean.Mul(size / (size - 1.0));
+      targetMean.Mul(size / (size - 1.0));
+      attributeMean.Add(-attributeValue / (size - 1.0));
+      targetMean.Add(-targetValue / (size - 1.0));
+      var dx = attributeValue - attributeMean.Get();
+      var dy = targetValue - targetMean.Get();
+      attributeVarSum.Add(-dx * dx2);
+      targetVarSum.Add(-dy * dy2);
+      comoment.Add(-dx * dy2);
+      size--;
+
+      ssr.Add(-predictOld * Predict(attributeValue, targetValue));
+    }
+
+    private double Predict(double attributeValue, double targetValue) {
+      return targetValue - Alpha - Beta * attributeValue;
+    }
+  }
+}
