Index: /branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj
===================================================================
--- /branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj	(revision 14884)
+++ /branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj	(revision 14885)
@@ -307,4 +307,14 @@
     <Compile Include="Interfaces\ISupportVectorMachineSolution.cs" />
     <Compile Include="Interfaces\IDataAnalysisAlgorithm.cs" />
+    <Compile Include="KernelRidgeRegression\KernelFunctions\CicularKernel.cs" />
+    <Compile Include="KernelRidgeRegression\KernelFunctions\GaussianKernel.cs" />
+    <Compile Include="KernelRidgeRegression\KernelFunctions\InverseMultiquadraticKernel.cs" />
+    <Compile Include="KernelRidgeRegression\KernelFunctions\KernelBase.cs" />
+    <Compile Include="KernelRidgeRegression\KernelFunctions\LaplacianKernel.cs" />
+    <Compile Include="KernelRidgeRegression\KernelFunctions\MultiquadraticKernel.cs" />
+    <Compile Include="KernelRidgeRegression\KernelFunctions\PolysplineKernel.cs" />
+    <Compile Include="KernelRidgeRegression\KernelFunctions\ThinPlatePolysplineKernel.cs" />
+    <Compile Include="KernelRidgeRegression\KernelRidgeRegression.cs" />
+    <Compile Include="KernelRidgeRegression\KernelRidgeRegressionModel.cs" />
     <Compile Include="kMeans\KMeansClustering.cs" />
     <Compile Include="kMeans\KMeansClusteringModel.cs" />
@@ -369,14 +379,4 @@
     <Compile Include="Plugin.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
-    <Compile Include="RadialBasisFunctions\KernelFunctions\CicularKernel.cs" />
-    <Compile Include="RadialBasisFunctions\KernelFunctions\GaussianKernel.cs" />
-    <Compile Include="RadialBasisFunctions\KernelFunctions\InverseMultiquadraticKernel.cs" />
-    <Compile Include="RadialBasisFunctions\KernelFunctions\KernelBase.cs" />
-    <Compile Include="RadialBasisFunctions\KernelFunctions\LaplacianKernel.cs" />
-    <Compile Include="RadialBasisFunctions\KernelFunctions\MultiquadraticKernel.cs" />
-    <Compile Include="RadialBasisFunctions\KernelFunctions\PolysplineKernel.cs" />
-    <Compile Include="RadialBasisFunctions\KernelFunctions\ThinPlatePolysplineKernel.cs" />
-    <Compile Include="RadialBasisFunctions\RadialBasisFunctionModel.cs" />
-    <Compile Include="RadialBasisFunctions\RadialBasisRegression.cs" />
     <Compile Include="RandomForest\RandomForestClassificationSolution.cs" />
     <Compile Include="RandomForest\RandomForestClassification.cs" />
@@ -402,4 +402,5 @@
     <Compile Include="TSNE\Distances\IndexedItemDistance.cs" />
     <Compile Include="TSNE\Distances\InnerProductDistance.cs" />
+    <Compile Include="TSNE\Distances\ManhattanDistance.cs" />
     <Compile Include="TSNE\Interfaces\IDistance.cs" />
     <Compile Include="TSNE\Interfaces\ISpacePartitioningTree.cs" />
@@ -434,8 +435,5 @@
     </BootstrapperPackage>
   </ItemGroup>
-  <ItemGroup>
-    <Folder Include="RadialBasisFunctions\Distances\" />
-    <Folder Include="RadialBasisFunctions\Interfaces\" />
-  </ItemGroup>
+  <ItemGroup />
   <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
Index: anches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/Distances/ManhattanDistance.cs
===================================================================
--- /branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/Distances/ManhattanDistance.cs	(revision 14884)
+++ 	(revision )
@@ -1,74 +1,0 @@
-﻿#region License Information
-/* HeuristicLab
- * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
- *
- * This file is part of HeuristicLab.
- *
- * HeuristicLab is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * HeuristicLab is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
- */
-#endregion
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using HeuristicLab.Common;
-using HeuristicLab.Core;
-using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-
-namespace HeuristicLab.Algorithms.DataAnalysis {
-  [StorableClass]
-  [Item("ManhattanDistance", "A distance function that uses block distance")]
-  public class ManhattanDistance : DistanceBase<IEnumerable<double>> {
-
-    #region HLConstructors & Boilerplate
-    [StorableConstructor]
-    protected ManhattanDistance(bool deserializing) : base(deserializing) { }
-    [StorableHook(HookType.AfterDeserialization)]
-    private void AfterDeserialization() { }
-    protected ManhattanDistance(ManhattanDistance original, Cloner cloner)
-      : base(original, cloner) { }
-    public ManhattanDistance() { }
-    public override IDeepCloneable Clone(Cloner cloner) {
-      return new ManhattanDistance(this, cloner);
-    }
-    #endregion
-
-    public static double GetDistance(double[] point1, double[] point2) {
-      if (point1.Length != point2.Length) throw new ArgumentException("Manhattan distance not defined on vectors of different length");
-      return point1.Zip(point2, (a1, b1) => Math.Abs(a1 - b1)).Sum();
-    }
-
-    public static double GetDistance(IEnumerable<double> a, IEnumerable<double> b, double threshold) {
-      double sum = 0;
-      var it1 = a.GetEnumerator();
-      var it2 = b.GetEnumerator();
-      while (sum > threshold && it1.MoveNext() && it2.MoveNext()) {
-        sum += Math.Abs(it1.Current - it2.Current);
-      }
-      it1.Dispose();
-      it2.Dispose();
-      return sum;
-    }
-
-    public override double Get(IEnumerable<double> a, IEnumerable<double> b) {
-      return GetDistance(a.ToArray(), b.ToArray());
-    }
-
-    public override double Get(IEnumerable<double> a, IEnumerable<double> b, double threshold) {
-      return GetDistance(a, b, threshold);
-    }
-
-
-  }
-}
Index: /branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegression.cs
===================================================================
--- /branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegression.cs	(revision 14885)
+++ /branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegression.cs	(revision 14885)
@@ -0,0 +1,120 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Linear regression data analysis algorithm.
+  /// </summary>
+  [Item("Radial Basis Function Regression (RBF-R)", "Radial basis function regression data analysis algorithm.")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 100)]
+  [StorableClass]
+  public sealed class RadialBasisRegression : BasicAlgorithm {
+    private const string RBFRegressionSolutionResultName = "RBF regression solution";
+
+    public override bool SupportsPause {
+      get { return false; }
+    }
+    public override Type ProblemType {
+      get { return typeof(IRegressionProblem); }
+    }
+    public new IRegressionProblem Problem {
+      get { return (IRegressionProblem)base.Problem; }
+      set { base.Problem = value; }
+    }
+
+    #region parameter names
+    private const string KernelParameterName = "Kernel";
+    private const string ScaleInputVariablesParameterName = "ScaleInputVariables";
+    #endregion
+
+    #region parameter properties
+    public ValueParameter<ICovarianceFunction> KernelParameter {
+      get { return (ValueParameter<ICovarianceFunction>)Parameters[KernelParameterName]; }
+    }
+
+    public IFixedValueParameter<BoolValue> ScaleInputVariablesParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[ScaleInputVariablesParameterName]; }
+    }
+    #endregion
+
+    #region properties
+    public ICovarianceFunction Kernel {
+      get { return KernelParameter.Value; }
+    }
+
+    public bool ScaleInputVariables {
+      get { return ScaleInputVariablesParameter.Value.Value; }
+      set { ScaleInputVariablesParameter.Value.Value = value; }
+    }
+
+    #endregion
+
+    [StorableConstructor]
+    private RadialBasisRegression(bool deserializing) : base(deserializing) { }
+    private RadialBasisRegression(RadialBasisRegression original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public RadialBasisRegression() {
+      Problem = new RegressionProblem();
+      Parameters.Add(new ValueParameter<ICovarianceFunction>(KernelParameterName, "The radial basis function"));
+      Parameters.Add(new FixedValueParameter<BoolValue>(ScaleInputVariablesParameterName, "Set to true if the input variables should be scaled to the interval [0..1]", new BoolValue(true)));
+      var kernel = new GaussianKernel();
+      KernelParameter.Value = kernel;
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RadialBasisRegression(this, cloner);
+    }
+
+    protected override void Run(CancellationToken cancellationToken) {
+      double loocvrmse, rmsError;
+      var solution = CreateRadialBasisRegressionSolution(Problem.ProblemData, Kernel, ScaleInputVariables, out loocvrmse, out rmsError);
+      Results.Add(new Result(RBFRegressionSolutionResultName, "The RBF regression solution.", solution));
+      Results.Add(new Result("LOOCVRMSE", "The root mean squared error of a leave-one-out-cross-validation on the training set", new DoubleValue(loocvrmse)));
+      Results.Add(new Result("RMSE (test)", "The root mean squared error of the solution on the test set.", new DoubleValue(rmsError)));
+    }
+
+    public static IRegressionSolution CreateRadialBasisRegressionSolution(IRegressionProblemData problemData, ICovarianceFunction kernel, bool scaleInputs, out double loocvRmsError, out double rmsError) {
+      var model = new RadialBasisFunctionModel(problemData.Dataset, problemData.TargetVariable, problemData.AllowedInputVariables, problemData.TrainingIndices, scaleInputs, kernel);
+      loocvRmsError = model.LeaveOneOutCrossValidationRootMeanSquaredError();
+      rmsError = Math.Sqrt(model.GetEstimatedValues(problemData.Dataset, problemData.TestIndices)
+        .Zip(problemData.TargetVariableTestValues, (a, b) => (a - b) * (a - b))
+        .Average());
+      var solution = model.CreateRegressionSolution((IRegressionProblemData)problemData.Clone());
+      solution.Model.Name = "RBF Regression Model";
+      solution.Name = "RBF Regression Solution";
+      return solution;
+    }
+  }
+}
Index: /branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegressionModel.cs
===================================================================
--- /branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegressionModel.cs	(revision 14885)
+++ /branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegressionModel.cs	(revision 14885)
@@ -0,0 +1,238 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents an RBF regression model.
+  /// </summary>
+  [StorableClass]
+  [Item("RBFModel", "An RBF regression model")]
+  public sealed class RadialBasisFunctionModel : RegressionModel, IConfidenceRegressionModel {
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return allowedInputVariables; }
+    }
+
+    [Storable]
+    private readonly string[] allowedInputVariables;
+    public string[] AllowedInputVariables {
+      get { return allowedInputVariables; }
+    }
+
+    [Storable]
+    private readonly double[] alpha;
+
+    [Storable]
+    private readonly double[,] trainX; // it is better to store the original training dataset completely because this is more efficient in persistence
+
+    [Storable]
+    private readonly ITransformation<double>[] scaling;
+
+    [Storable]
+    private readonly ICovarianceFunction kernel;
+
+    private double[,] gramInvert; // not storable as it can be large (recreate after deserialization as required)
+
+    [Storable]
+    private readonly double meanOffset; // implementation works for zero-mean target variables
+
+    [StorableConstructor]
+    private RadialBasisFunctionModel(bool deserializing) : base(deserializing) { }
+    private RadialBasisFunctionModel(RadialBasisFunctionModel original, Cloner cloner)
+      : base(original, cloner) {
+      // shallow copies of arrays because they cannot be modified
+      allowedInputVariables = original.allowedInputVariables;
+      alpha = original.alpha;
+      trainX = original.trainX;
+      gramInvert = original.gramInvert;
+      scaling = original.scaling;
+
+      meanOffset = original.meanOffset;
+      if (original.kernel != null)
+        kernel = cloner.Clone(original.kernel);
+    }
+    public RadialBasisFunctionModel(IDataset dataset, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<int> rows,
+      bool scaleInputs, ICovarianceFunction kernel)
+      : base(targetVariable) {
+      if (kernel.GetNumberOfParameters(allowedInputVariables.Count()) > 0) throw new ArgumentException("All parameters in the kernel function must be specified.");
+      name = ItemName;
+      description = ItemDescription;
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+      var trainingRows = rows.ToArray();
+      this.kernel = (ICovarianceFunction)kernel.Clone();
+      try {
+        if (scaleInputs)
+          scaling = CreateScaling(dataset, trainingRows);
+        trainX = ExtractData(dataset, trainingRows, scaling);
+        var y = dataset.GetDoubleValues(targetVariable, trainingRows).ToArray();
+        meanOffset = y.Average();
+        for (int i = 0; i < y.Length; i++) y[i] -= meanOffset;
+        int info;
+        // TODO: improve efficiency by decomposing matrix once instead of solving the system and then inverting the matrix
+        alglib.densesolverlsreport denseSolveRep;
+        gramInvert = BuildGramMatrix(trainX);
+        int n = trainX.GetLength(0);
+        alglib.rmatrixsolvels(gramInvert, n, n, y, 0.0, out info, out denseSolveRep, out alpha);
+        if (info != 1) throw new ArgumentException("Could not create model.");
+
+        alglib.matinvreport report;
+        alglib.rmatrixinverse(ref gramInvert, out info, out report);
+        if (info != 1) throw new ArgumentException("Could not invert matrix. Is it quadratic symmetric positive definite?");
+
+      } catch (alglib.alglibexception ae) {
+        // wrap exception so that calling code doesn't have to know about alglib implementation
+        throw new ArgumentException("There was a problem in the calculation of the RBF model", ae);
+      }
+    }
+
+    private ITransformation<double>[] CreateScaling(IDataset dataset, int[] rows) {
+      var trans = new ITransformation<double>[allowedInputVariables.Length];
+      int i = 0;
+      foreach (var variable in allowedInputVariables) {
+        var lin = new LinearTransformation(allowedInputVariables);
+        var max = dataset.GetDoubleValues(variable, rows).Max();
+        var min = dataset.GetDoubleValues(variable, rows).Min();
+        lin.Multiplier = 1.0 / (max - min);
+        lin.Addend = -min / (max - min);
+        trans[i] = lin;
+        i++;
+      }
+      return trans;
+    }
+
+    private double[,] ExtractData(IDataset dataset, IEnumerable<int> rows, ITransformation<double>[] scaling = null) {
+      double[][] variables;
+      if (scaling != null) {
+        variables =
+          allowedInputVariables.Select((var, i) => scaling[i].Apply(dataset.GetDoubleValues(var, rows)).ToArray())
+            .ToArray();
+      } else {
+        variables =
+        allowedInputVariables.Select(var => dataset.GetDoubleValues(var, rows).ToArray()).ToArray();
+      }
+      int n = variables.First().Length;
+      var res = new double[n, variables.Length];
+      for (int r = 0; r < n; r++)
+        for (int c = 0; c < variables.Length; c++) {
+          res[r, c] = variables[c][r];
+        }
+      return res;
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RadialBasisFunctionModel(this, cloner);
+    }
+
+    #region IRegressionModel Members
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      var newX = ExtractData(dataset, rows, scaling);
+      var dim = newX.GetLength(1);
+      var cov = kernel.GetParameterizedCovarianceFunction(new double[0], Enumerable.Range(0, dim).ToArray());
+
+      var pred = new double[newX.GetLength(0)];
+      for (int i = 0; i < pred.Length; i++) {
+        double sum = meanOffset;
+        for (int j = 0; j < alpha.Length; j++) {
+          sum += alpha[j] * cov.CrossCovariance(trainX, newX, j, i);
+        }
+        pred[i] = sum;
+      }
+      return pred;
+    }
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new ConfidenceRegressionSolution(this, new RegressionProblemData(problemData));
+    }
+    #endregion
+
+    public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
+      if (gramInvert == null) CreateAndInvertGramMatrix();
+      int n = gramInvert.GetLength(0);
+      var newData = ExtractData(dataset, rows, scaling);
+      var dim = newData.GetLength(1);
+      var cov = kernel.GetParameterizedCovarianceFunction(new double[0], Enumerable.Range(0, dim).ToArray());
+
+      // TODO perf (matrix matrix multiplication)
+      for (int i = 0; i < newData.GetLength(0); i++) {
+        double[] p = new double[n];
+
+        for (int j = 0; j < trainX.GetLength(0); j++) {
+          p[j] = cov.CrossCovariance(trainX, newData, j, i);
+        }
+
+        var Ap = new double[n];
+        alglib.ablas.rmatrixmv(n, n, gramInvert, 0, 0, 0, p, 0, ref Ap, 0);
+        var res = 0.0;
+        // dot product
+        for (int j = 0; j < p.Length; j++) res += p[j] * Ap[j];
+        yield return res > 0 ? res : 0;
+      }
+    }
+    public double LeaveOneOutCrossValidationRootMeanSquaredError() {
+      if (gramInvert == null) CreateAndInvertGramMatrix();
+      var n = gramInvert.GetLength(0);
+      var s = 1.0 / n;
+
+      var sum = 0.0;
+      for (int i = 0; i < alpha.Length; i++) {
+        var x = alpha[i] / gramInvert[i, i];
+        sum += x * x;
+      }
+      sum *= s;
+      return Math.Sqrt(sum);
+    }
+
+    private void CreateAndInvertGramMatrix() {
+      try {
+        gramInvert = BuildGramMatrix(trainX);
+        int info = 0;
+        alglib.matinvreport report;
+        alglib.rmatrixinverse(ref gramInvert, out info, out report);
+        if (info != 1)
+          throw new ArgumentException("Could not invert matrix. Is it quadratic symmetric positive definite?");
+      } catch (alglib.alglibexception) {
+        // wrap exception so that calling code doesn't have to know about alglib implementation
+        throw new ArgumentException("Could not invert matrix. Is it quadratic symmetric positive definite?");
+      }
+    }
+    #region helpers
+    private double[,] BuildGramMatrix(double[,] data) {
+      var n = data.GetLength(0);
+      var dim = data.GetLength(1);
+      var cov = kernel.GetParameterizedCovarianceFunction(new double[0], Enumerable.Range(0, dim).ToArray());
+      var gram = new double[n, n];
+      for (var i = 0; i < n; i++)
+        for (var j = i; j < n; j++) {
+          gram[j, i] = gram[i, j] = cov.Covariance(data, i, j); // symmetric matrix --> half of the work
+        }
+      return gram;
+    }
+
+    #endregion
+  }
+}
Index: anches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/RadialBasisFunctionModel.cs
===================================================================
--- /branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/RadialBasisFunctionModel.cs	(revision 14884)
+++ 	(revision )
@@ -1,238 +1,0 @@
-﻿#region License Information
-/* HeuristicLab
- * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
- *
- * This file is part of HeuristicLab.
- *
- * HeuristicLab is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * HeuristicLab is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
- */
-#endregion
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using HeuristicLab.Common;
-using HeuristicLab.Core;
-using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using HeuristicLab.Problems.DataAnalysis;
-
-namespace HeuristicLab.Algorithms.DataAnalysis {
-  /// <summary>
-  /// Represents an RBF regression model.
-  /// </summary>
-  [StorableClass]
-  [Item("RBFModel", "An RBF regression model")]
-  public sealed class RadialBasisFunctionModel : RegressionModel, IConfidenceRegressionModel {
-    public override IEnumerable<string> VariablesUsedForPrediction {
-      get { return allowedInputVariables; }
-    }
-
-    [Storable]
-    private readonly string[] allowedInputVariables;
-    public string[] AllowedInputVariables {
-      get { return allowedInputVariables; }
-    }
-
-    [Storable]
-    private readonly double[] alpha;
-
-    [Storable]
-    private readonly double[,] trainX; // it is better to store the original training dataset completely because this is more efficient in persistence
-
-    [Storable]
-    private readonly ITransformation<double>[] scaling;
-
-    [Storable]
-    private readonly ICovarianceFunction kernel;
-
-    private double[,] gramInvert; // not storable as it can be large (recreate after deserialization as required)
-
-    [Storable]
-    private readonly double meanOffset; // implementation works for zero-mean target variables
-
-    [StorableConstructor]
-    private RadialBasisFunctionModel(bool deserializing) : base(deserializing) { }
-    private RadialBasisFunctionModel(RadialBasisFunctionModel original, Cloner cloner)
-      : base(original, cloner) {
-      // shallow copies of arrays because they cannot be modified
-      allowedInputVariables = original.allowedInputVariables;
-      alpha = original.alpha;
-      trainX = original.trainX;
-      gramInvert = original.gramInvert;
-      scaling = original.scaling;
-
-      meanOffset = original.meanOffset;
-      if (original.kernel != null)
-        kernel = cloner.Clone(original.kernel);
-    }
-    public RadialBasisFunctionModel(IDataset dataset, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<int> rows,
-      bool scaleInputs, ICovarianceFunction kernel)
-      : base(targetVariable) {
-      if (kernel.GetNumberOfParameters(allowedInputVariables.Count()) > 0) throw new ArgumentException("All parameters in the kernel function must be specified.");
-      name = ItemName;
-      description = ItemDescription;
-      this.allowedInputVariables = allowedInputVariables.ToArray();
-      var trainingRows = rows.ToArray();
-      this.kernel = (ICovarianceFunction)kernel.Clone();
-      try {
-        if (scaleInputs)
-          scaling = CreateScaling(dataset, trainingRows);
-        trainX = ExtractData(dataset, trainingRows, scaling);
-        var y = dataset.GetDoubleValues(targetVariable, trainingRows).ToArray();
-        meanOffset = y.Average();
-        for (int i = 0; i < y.Length; i++) y[i] -= meanOffset;
-        int info;
-        // TODO: improve efficiency by decomposing matrix once instead of solving the system and then inverting the matrix
-        alglib.densesolverlsreport denseSolveRep;
-        gramInvert = BuildGramMatrix(trainX);
-        int n = trainX.GetLength(0);
-        alglib.rmatrixsolvels(gramInvert, n, n, y, 0.0, out info, out denseSolveRep, out alpha);
-        if (info != 1) throw new ArgumentException("Could not create model.");
-
-        alglib.matinvreport report;
-        alglib.rmatrixinverse(ref gramInvert, out info, out report);
-        if (info != 1) throw new ArgumentException("Could not invert matrix. Is it quadratic symmetric positive definite?");
-
-      } catch (alglib.alglibexception ae) {
-        // wrap exception so that calling code doesn't have to know about alglib implementation
-        throw new ArgumentException("There was a problem in the calculation of the RBF model", ae);
-      }
-    }
-
-    private ITransformation<double>[] CreateScaling(IDataset dataset, int[] rows) {
-      var trans = new ITransformation<double>[allowedInputVariables.Length];
-      int i = 0;
-      foreach (var variable in allowedInputVariables) {
-        var lin = new LinearTransformation(allowedInputVariables);
-        var max = dataset.GetDoubleValues(variable, rows).Max();
-        var min = dataset.GetDoubleValues(variable, rows).Min();
-        lin.Multiplier = 1.0 / (max - min);
-        lin.Addend = -min / (max - min);
-        trans[i] = lin;
-        i++;
-      }
-      return trans;
-    }
-
-    private double[,] ExtractData(IDataset dataset, IEnumerable<int> rows, ITransformation<double>[] scaling = null) {
-      double[][] variables;
-      if (scaling != null) {
-        variables =
-          allowedInputVariables.Select((var, i) => scaling[i].Apply(dataset.GetDoubleValues(var, rows)).ToArray())
-            .ToArray();
-      } else {
-        variables =
-        allowedInputVariables.Select(var => dataset.GetDoubleValues(var, rows).ToArray()).ToArray();
-      }
-      int n = variables.First().Length;
-      var res = new double[n, variables.Length];
-      for (int r = 0; r < n; r++)
-        for (int c = 0; c < variables.Length; c++) {
-          res[r, c] = variables[c][r];
-        }
-      return res;
-    }
-
-    public override IDeepCloneable Clone(Cloner cloner) {
-      return new RadialBasisFunctionModel(this, cloner);
-    }
-
-    #region IRegressionModel Members
-    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
-      var newX = ExtractData(dataset, rows, scaling);
-      var dim = newX.GetLength(1);
-      var cov = kernel.GetParameterizedCovarianceFunction(new double[0], Enumerable.Range(0, dim).ToArray());
-
-      var pred = new double[newX.GetLength(0)];
-      for (int i = 0; i < pred.Length; i++) {
-        double sum = meanOffset;
-        for (int j = 0; j < alpha.Length; j++) {
-          sum += alpha[j] * cov.CrossCovariance(trainX, newX, j, i);
-        }
-        pred[i] = sum;
-      }
-      return pred;
-    }
-    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
-      return new ConfidenceRegressionSolution(this, new RegressionProblemData(problemData));
-    }
-    #endregion
-
-    public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
-      if (gramInvert == null) CreateAndInvertGramMatrix();
-      int n = gramInvert.GetLength(0);
-      var newData = ExtractData(dataset, rows, scaling);
-      var dim = newData.GetLength(1);
-      var cov = kernel.GetParameterizedCovarianceFunction(new double[0], Enumerable.Range(0, dim).ToArray());
-
-      // TODO perf (matrix matrix multiplication)
-      for (int i = 0; i < newData.GetLength(0); i++) {
-        double[] p = new double[n];
-
-        for (int j = 0; j < trainX.GetLength(0); j++) {
-          p[j] = cov.CrossCovariance(trainX, newData, j, i);
-        }
-
-        var Ap = new double[n];
-        alglib.ablas.rmatrixmv(n, n, gramInvert, 0, 0, 0, p, 0, ref Ap, 0);
-        var res = 0.0;
-        // dot product
-        for (int j = 0; j < p.Length; j++) res += p[j] * Ap[j];
-        yield return res > 0 ? res : 0;
-      }
-    }
-    public double LeaveOneOutCrossValidationRootMeanSquaredError() {
-      if (gramInvert == null) CreateAndInvertGramMatrix();
-      var n = gramInvert.GetLength(0);
-      var s = 1.0 / n;
-
-      var sum = 0.0;
-      for (int i = 0; i < alpha.Length; i++) {
-        var x = alpha[i] / gramInvert[i, i];
-        sum += x * x;
-      }
-      sum *= s;
-      return Math.Sqrt(sum);
-    }
-
-    private void CreateAndInvertGramMatrix() {
-      try {
-        gramInvert = BuildGramMatrix(trainX);
-        int info = 0;
-        alglib.matinvreport report;
-        alglib.rmatrixinverse(ref gramInvert, out info, out report);
-        if (info != 1)
-          throw new ArgumentException("Could not invert matrix. Is it quadratic symmetric positive definite?");
-      } catch (alglib.alglibexception) {
-        // wrap exception so that calling code doesn't have to know about alglib implementation
-        throw new ArgumentException("Could not invert matrix. Is it quadratic symmetric positive definite?");
-      }
-    }
-    #region helpers
-    private double[,] BuildGramMatrix(double[,] data) {
-      var n = data.GetLength(0);
-      var dim = data.GetLength(1);
-      var cov = kernel.GetParameterizedCovarianceFunction(new double[0], Enumerable.Range(0, dim).ToArray());
-      var gram = new double[n, n];
-      for (var i = 0; i < n; i++)
-        for (var j = i; j < n; j++) {
-          gram[j, i] = gram[i, j] = cov.Covariance(data, i, j); // symmetric matrix --> half of the work
-        }
-      return gram;
-    }
-
-    #endregion
-  }
-}
Index: anches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/RadialBasisRegression.cs
===================================================================
--- /branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/RadialBasisRegression.cs	(revision 14884)
+++ 	(revision )
@@ -1,120 +1,0 @@
-#region License Information
-/* HeuristicLab
- * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
- *
- * This file is part of HeuristicLab.
- *
- * HeuristicLab is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * HeuristicLab is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
- */
-#endregion
-
-using System;
-using System.Linq;
-using System.Threading;
-using HeuristicLab.Common;
-using HeuristicLab.Core;
-using HeuristicLab.Data;
-using HeuristicLab.Optimization;
-using HeuristicLab.Parameters;
-using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using HeuristicLab.Problems.DataAnalysis;
-
-namespace HeuristicLab.Algorithms.DataAnalysis {
-  /// <summary>
-  /// Linear regression data analysis algorithm.
-  /// </summary>
-  [Item("Radial Basis Function Regression (RBF-R)", "Radial basis function regression data analysis algorithm.")]
-  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 100)]
-  [StorableClass]
-  public sealed class RadialBasisRegression : BasicAlgorithm {
-    private const string RBFRegressionSolutionResultName = "RBF regression solution";
-
-    public override bool SupportsPause {
-      get { return false; }
-    }
-    public override Type ProblemType {
-      get { return typeof(IRegressionProblem); }
-    }
-    public new IRegressionProblem Problem {
-      get { return (IRegressionProblem)base.Problem; }
-      set { base.Problem = value; }
-    }
-
-    #region parameter names
-    private const string KernelParameterName = "Kernel";
-    private const string ScaleInputVariablesParameterName = "ScaleInputVariables";
-    #endregion
-
-    #region parameter properties
-    public ValueParameter<ICovarianceFunction> KernelParameter {
-      get { return (ValueParameter<ICovarianceFunction>)Parameters[KernelParameterName]; }
-    }
-
-    public IFixedValueParameter<BoolValue> ScaleInputVariablesParameter {
-      get { return (IFixedValueParameter<BoolValue>)Parameters[ScaleInputVariablesParameterName]; }
-    }
-    #endregion
-
-    #region properties
-    public ICovarianceFunction Kernel {
-      get { return KernelParameter.Value; }
-    }
-
-    public bool ScaleInputVariables {
-      get { return ScaleInputVariablesParameter.Value.Value; }
-      set { ScaleInputVariablesParameter.Value.Value = value; }
-    }
-
-    #endregion
-
-    [StorableConstructor]
-    private RadialBasisRegression(bool deserializing) : base(deserializing) { }
-    private RadialBasisRegression(RadialBasisRegression original, Cloner cloner)
-      : base(original, cloner) {
-    }
-    public RadialBasisRegression() {
-      Problem = new RegressionProblem();
-      Parameters.Add(new ValueParameter<ICovarianceFunction>(KernelParameterName, "The radial basis function"));
-      Parameters.Add(new FixedValueParameter<BoolValue>(ScaleInputVariablesParameterName, "Set to true if the input variables should be scaled to the interval [0..1]", new BoolValue(true)));
-      var kernel = new GaussianKernel();
-      KernelParameter.Value = kernel;
-    }
-    [StorableHook(HookType.AfterDeserialization)]
-    private void AfterDeserialization() { }
-
-    public override IDeepCloneable Clone(Cloner cloner) {
-      return new RadialBasisRegression(this, cloner);
-    }
-
-    protected override void Run(CancellationToken cancellationToken) {
-      double loocvrmse, rmsError;
-      var solution = CreateRadialBasisRegressionSolution(Problem.ProblemData, Kernel, ScaleInputVariables, out loocvrmse, out rmsError);
-      Results.Add(new Result(RBFRegressionSolutionResultName, "The RBF regression solution.", solution));
-      Results.Add(new Result("LOOCVRMSE", "The root mean squared error of a leave-one-out-cross-validation on the training set", new DoubleValue(loocvrmse)));
-      Results.Add(new Result("RMSE (test)", "The root mean squared error of the solution on the test set.", new DoubleValue(rmsError)));
-    }
-
-    public static IRegressionSolution CreateRadialBasisRegressionSolution(IRegressionProblemData problemData, ICovarianceFunction kernel, bool scaleInputs, out double loocvRmsError, out double rmsError) {
-      var model = new RadialBasisFunctionModel(problemData.Dataset, problemData.TargetVariable, problemData.AllowedInputVariables, problemData.TrainingIndices, scaleInputs, kernel);
-      loocvRmsError = model.LeaveOneOutCrossValidationRootMeanSquaredError();
-      rmsError = Math.Sqrt(model.GetEstimatedValues(problemData.Dataset, problemData.TestIndices)
-        .Zip(problemData.TargetVariableTestValues, (a, b) => (a - b) * (a - b))
-        .Average());
-      var solution = model.CreateRegressionSolution((IRegressionProblemData)problemData.Clone());
-      solution.Model.Name = "RBF Regression Model";
-      solution.Name = "RBF Regression Solution";
-      return solution;
-    }
-  }
-}
Index: /branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/ManhattanDistance.cs
===================================================================
--- /branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/ManhattanDistance.cs	(revision 14885)
+++ /branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/Distances/ManhattanDistance.cs	(revision 14885)
@@ -0,0 +1,56 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item("ManhattanDistance", "A distance function that uses block distance")]
+  public class ManhattanDistance : DistanceBase<IEnumerable<double>> {
+
+    #region HLConstructors & Boilerplate
+    [StorableConstructor]
+    protected ManhattanDistance(bool deserializing) : base(deserializing) { }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+    protected ManhattanDistance(ManhattanDistance original, Cloner cloner)
+      : base(original, cloner) { }
+    public ManhattanDistance() { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new ManhattanDistance(this, cloner);
+    }
+    #endregion
+
+    public static double GetDistance(double[] point1, double[] point2) {
+      if (point1.Length != point2.Length) throw new ArgumentException("Manhattan distance not defined on vectors of different length");
+      return point1.Zip(point2, (a1, b1) => Math.Abs(a1 - b1)).Sum();
+    }
+
+    public override double Get(IEnumerable<double> a, IEnumerable<double> b) {
+      return GetDistance(a.ToArray(), b.ToArray());
+    }
+  }
+}
