Index: branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj
===================================================================
--- branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj	(revision 14885)
+++ branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj	(revision 14887)
@@ -309,7 +309,7 @@
     <Compile Include="KernelRidgeRegression\KernelFunctions\CicularKernel.cs" />
     <Compile Include="KernelRidgeRegression\KernelFunctions\GaussianKernel.cs" />
+    <Compile Include="KernelRidgeRegression\KernelFunctions\IKernel.cs" />
     <Compile Include="KernelRidgeRegression\KernelFunctions\InverseMultiquadraticKernel.cs" />
     <Compile Include="KernelRidgeRegression\KernelFunctions\KernelBase.cs" />
-    <Compile Include="KernelRidgeRegression\KernelFunctions\LaplacianKernel.cs" />
     <Compile Include="KernelRidgeRegression\KernelFunctions\MultiquadraticKernel.cs" />
     <Compile Include="KernelRidgeRegression\KernelFunctions\PolysplineKernel.cs" />
Index: branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/CicularKernel.cs
===================================================================
--- branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/CicularKernel.cs	(revision 14885)
+++ branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/CicularKernel.cs	(revision 14887)
@@ -23,11 +23,9 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
-using HeuristicLab.Data;
-using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
-namespace HeuristicLab.Algorithms.DataAnalysis {
+namespace HeuristicLab.Algorithms.DataAnalysis.KernelRidgeRegression {
   [StorableClass]
-  [Item("CircularKernel", "A circular kernel function")]
+  [Item("CircularKernel", "A circular kernel function 2*pi*(acos(-d)-d*(1-n²)^(0.5)) where n = ||x-c|| and d = n/beta")]
   public class CircularKernel : KernelBase {
 
@@ -39,5 +37,4 @@
     protected CircularKernel(CircularKernel original, Cloner cloner) : base(original, cloner) { }
     public CircularKernel() {
-      Parameters.Add(new FixedValueParameter<DoubleValue>(BetaParameterName, "The beta in the kernel function 2*pi*(acos(-d)-d*(1-n²)^(0.5)) where n = ||x-c|| and d = n/beta", new DoubleValue(2)));
     }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -47,14 +44,16 @@
 
     protected override double Get(double norm) {
-      if (Math.Abs(Beta) < double.Epsilon) return double.NaN;
-      if (norm >= Beta) return 0;
-      var d = norm / Beta;
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      if (norm >= beta) return 0;
+      var d = norm / beta;
       return Math.Acos(-d) - d * Math.Sqrt(1 - d * d) - Math.PI / 2;
     }
 
     protected override double GetGradient(double norm) {
-      if (Math.Abs(Beta) < double.Epsilon) return double.NaN;
-      if (Beta < norm) return 0;
-      return -2*Math.Pow(norm,3)/(Math.Pow(Beta,4)*Math.Sqrt(1-norm*norm/(Beta*Beta)));
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      if (beta < norm) return 0;
+      return -2 * Math.Pow(norm, 3) / (Math.Pow(beta, 4) * Math.Sqrt(1 - norm * norm / (beta * beta)));
     }
   }
Index: branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/GaussianKernel.cs
===================================================================
--- branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/GaussianKernel.cs	(revision 14885)
+++ branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/GaussianKernel.cs	(revision 14887)
@@ -21,13 +21,13 @@
 
 using System;
+
 using HeuristicLab.Common;
 using HeuristicLab.Core;
-using HeuristicLab.Data;
-using HeuristicLab.Parameters;
+
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
-namespace HeuristicLab.Algorithms.DataAnalysis {
+namespace HeuristicLab.Algorithms.DataAnalysis.KernelRidgeRegression {
   [StorableClass]
-  [Item("GaussianKernel", "A kernel function that uses Gaussian function")]
+  [Item("GaussianKernel", "A kernel function that uses Gaussian function exp(-||x-c||/beta²). Positive definite beta > 0")]
   public class GaussianKernel : KernelBase {
 
@@ -39,5 +39,4 @@
     protected GaussianKernel(GaussianKernel original, Cloner cloner) : base(original, cloner) { }
     public GaussianKernel() {
-      Parameters.Add(new FixedValueParameter<DoubleValue>(BetaParameterName, "The beta in the kernelfunction exp(-||x-c||/beta²)", new DoubleValue(2)));
     }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -47,11 +46,13 @@
 
     protected override double Get(double norm) {
-      if (Math.Abs(Beta) < double.Epsilon) return double.NaN;
-      return Math.Exp(-norm * norm / (Beta * Beta));
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      return Math.Exp(-norm * norm / (beta * beta));
     }
 
     protected override double GetGradient(double norm) {
-      if (Math.Abs(Beta) < double.Epsilon) return double.NaN;
-      return 2 * norm * norm / Math.Pow(Beta, 3) * Math.Exp(-norm * norm / (Beta * Beta));
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      return 2 * norm * norm / Math.Pow(beta, 3) * Math.Exp(-norm * norm / (beta * beta));
     }
   }
Index: branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/IKernel.cs
===================================================================
--- branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/IKernel.cs	(revision 14887)
+++ branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/IKernel.cs	(revision 14887)
@@ -0,0 +1,28 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+
+namespace HeuristicLab.Algorithms.DataAnalysis.KernelRidgeRegression {
+  public interface IKernel : ICovarianceFunction {
+    double? Beta { get; set; } // a kernel parameter
+    IDistance Distance { get; set; } // the distance function to use
+  }
+}
Index: branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/InverseMultiquadraticKernel.cs
===================================================================
--- branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/InverseMultiquadraticKernel.cs	(revision 14885)
+++ branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/InverseMultiquadraticKernel.cs	(revision 14887)
@@ -22,12 +22,10 @@
 using System;
 using HeuristicLab.Common;
-using HeuristicLab.Core;
-using HeuristicLab.Data;
-using HeuristicLab.Parameters;
+using HeuristicLab.Core;     
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
-namespace HeuristicLab.Algorithms.DataAnalysis {
+namespace HeuristicLab.Algorithms.DataAnalysis.KernelRidgeRegression {
   [StorableClass]
-  [Item("InverseMultiquadraticKernel", "A kernel function that uses the inverse multiquadratic function")]
+  [Item("InverseMultiquadraticKernel", "A kernel function that uses the inverse multi-quadratic function  1 / sqrt(1+||x-c||^2/beta). Positive definite: beta > 0")]
   public class InverseMultiquadraticKernel : KernelBase {
     #region HLConstructors & Boilerplate
@@ -38,5 +36,4 @@
     protected InverseMultiquadraticKernel(InverseMultiquadraticKernel original, Cloner cloner) : base(original, cloner) { }
     public InverseMultiquadraticKernel() {
-      Parameters.Add(new FixedValueParameter<DoubleValue>(BetaParameterName, "The beta in the kernel function 1 / sqrt(1+||x-c||^2/beta)", new DoubleValue(2)));
     }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -46,11 +43,13 @@
 
     protected override double Get(double norm) {
-      if (Math.Abs(Beta) < double.Epsilon) return double.NaN;
-      return 1 / Math.Sqrt(1 + norm * norm / Beta);
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      return 1 / Math.Sqrt(1 + norm * norm / beta);
     }
 
     protected override double GetGradient(double norm) {
-      if (Math.Abs(Beta) < double.Epsilon) return double.NaN;
-      return norm * norm / (2 * Beta * Beta * Math.Pow((norm * norm + Beta) / Beta, 1.5));
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      return norm * norm / (2 * beta * beta * Math.Pow((norm * norm + beta) / beta, 1.5));
     }
   }
Index: branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/KernelBase.cs
===================================================================
--- branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/KernelBase.cs	(revision 14885)
+++ branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/KernelBase.cs	(revision 14887)
@@ -21,19 +21,17 @@
 
 using System;
-using System.Collections;
 using System.Collections.Generic;
+using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
-using HeuristicLab.Data;
 using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
-namespace HeuristicLab.Algorithms.DataAnalysis {
+namespace HeuristicLab.Algorithms.DataAnalysis.KernelRidgeRegression {
   [StorableClass]
-  public abstract class KernelBase : ParameterizedNamedItem, ICovarianceFunction {
+  public abstract class KernelBase : ParameterizedNamedItem, IKernel {
 
     #region Parameternames
     private const string DistanceParameterName = "Distance";
-    protected const string BetaParameterName = "Beta";
     #endregion
     #region Parameterproperties
@@ -42,16 +40,11 @@
     }
 
-    public IFixedValueParameter<DoubleValue> BetaParameter {
-      get { return Parameters[BetaParameterName] as FixedValueParameter<DoubleValue>; }
-    }
-
+    [Storable]
+    public double? Beta { get; set; }
     #endregion
     #region Properties
     public IDistance Distance {
       get { return DistanceParameter.Value; }
-    }
-
-    public double Beta {
-      get { return BetaParameter.Value.Value; }
+      set { DistanceParameter.Value = value; }
     }
 
@@ -64,5 +57,7 @@
 
     protected KernelBase(KernelBase original, Cloner cloner)
-      : base(original, cloner) { }
+      : base(original, cloner) {
+      Beta = original.Beta;
+    }
 
     protected KernelBase() {
@@ -78,15 +73,15 @@
 
     public int GetNumberOfParameters(int numberOfVariables) {
-      return 1;
+      return Beta.HasValue ? 0 : 1;
     }
 
     public void SetParameter(double[] p) {
-      if (p != null && p.Length == 1) BetaParameter.Value.Value = p[0];
+      if (p != null && p.Length == 1) Beta = new double?(p[0]);
     }
 
     public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
-      if (p == null || p.Length != 1) throw new ArgumentException("Illegal parametrization");
+      if (p.Length != GetNumberOfParameters(columnIndices.Length)) throw new ArgumentException("Illegal parametrization");
       var myClone = (KernelBase)Clone(new Cloner());
-      myClone.BetaParameter.Value.Value = p[0];
+      myClone.SetParameter(p);
       var cov = new ParameterizedCovarianceFunction {
         Covariance = (x, i, j) => myClone.Get(GetNorm(x, x, i, j, columnIndices)),
@@ -102,57 +97,7 @@
       var dist = Distance as IDistance<IEnumerable<double>>;
       if (dist == null) throw new ArgumentException("The distance needs to apply to double vectors");
-      var r1 = new IndexedEnumerable(x, i, columnIndices);
-      var r2 = new IndexedEnumerable(xt, j, columnIndices);
+      var r1 = columnIndices.Select(c => x[i, c]);
+      var r2 = columnIndices.Select(c => xt[j, c]);
       return dist.Get(r1, r2);
-    }
-    internal class IndexedEnumerable : IEnumerable<double> {
-      private readonly double[,] data;
-      private readonly int row;
-      private readonly int[] columnIndices;
-
-      public IndexedEnumerable(double[,] data, int row, int[] columnIndices) {
-        this.data = data;
-        this.row = row;
-        this.columnIndices = columnIndices;
-      }
-
-      public IEnumerator<double> GetEnumerator() {
-        return new IndexedEnumerator(data, row, columnIndices);
-      }
-
-      IEnumerator IEnumerable.GetEnumerator() {
-        return new IndexedEnumerator(data, row, columnIndices);
-      }
-    }
-    internal class IndexedEnumerator : IEnumerator<double> {
-      private readonly IEnumerator<int> column;
-      private readonly double[,] data;
-      private readonly int row;
-
-      public IndexedEnumerator(double[,] data, int row, int[] columnIndices) {
-        this.data = data;
-        this.row = row;
-        column = ((IEnumerable<int>)columnIndices).GetEnumerator();
-      }
-
-      public double Current {
-        get { return data[row, column.Current]; }
-      }
-
-      object IEnumerator.Current {
-        get {
-          return data[row, column.Current];
-        }
-      }
-
-      public void Dispose() { }
-
-      public bool MoveNext() {
-        return column.MoveNext();
-      }
-
-      public void Reset() {
-        column.Reset();
-      }
     }
   }
Index: branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/LaplacianKernel.cs
===================================================================
--- branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/LaplacianKernel.cs	(revision 14885)
+++ 	(revision )
@@ -1,55 +1,0 @@
-﻿#region License Information
-/* HeuristicLab
- * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
- *
- * This file is part of HeuristicLab.
- *
- * HeuristicLab is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * HeuristicLab is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
- */
-#endregion
-
-using System;
-using HeuristicLab.Common;
-using HeuristicLab.Core;
-using HeuristicLab.Data;
-using HeuristicLab.Parameters;
-using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-
-namespace HeuristicLab.Algorithms.DataAnalysis {
-  [StorableClass]
-  [Item("LaplacianKernel", "A kernel function that uses an exponential function.\nIt is equvalent to the Gaussian kernel but less susceptible to improper values of beta")]
-  public class LaplacianKernel : KernelBase {
-    [StorableConstructor]
-    protected LaplacianKernel(bool deserializing) : base(deserializing) { }
-    [StorableHook(HookType.AfterDeserialization)]
-    private void AfterDeserialization() { }
-    protected LaplacianKernel(LaplacianKernel original, Cloner cloner) : base(original, cloner) { }
-    public LaplacianKernel() {
-      Parameters.Add(new FixedValueParameter<DoubleValue>(BetaParameterName, "The beta in the kernel function exp(-||x-c||/beta)", new DoubleValue(2)));
-    }
-    public override IDeepCloneable Clone(Cloner cloner) {
-      return new LaplacianKernel(this, cloner);
-    }
-
-    protected override double Get(double norm) {
-      if (Math.Abs(Beta) < double.Epsilon) return double.NaN;
-      return Math.Exp(-norm / Beta);
-    }
-
-    protected override double GetGradient(double norm) {
-      if (Math.Abs(Beta) < double.Epsilon) return double.NaN;
-      return norm * Math.Exp(-norm / Beta) / (Beta * Beta);
-    }
-  }
-}
Index: branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/MultiquadraticKernel.cs
===================================================================
--- branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/MultiquadraticKernel.cs	(revision 14885)
+++ branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/MultiquadraticKernel.cs	(revision 14887)
@@ -22,12 +22,11 @@
 using System;
 using HeuristicLab.Common;
-using HeuristicLab.Core;
-using HeuristicLab.Data;
-using HeuristicLab.Parameters;
+using HeuristicLab.Core;            
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
-namespace HeuristicLab.Algorithms.DataAnalysis {
+namespace HeuristicLab.Algorithms.DataAnalysis.KernelRidgeRegression {
   [StorableClass]
-  [Item("MultiquadraticKernel", "A kernel function that uses the multiquadratic function")]
+  // conditionally positive definite. (need to add polynomials) see http://num.math.uni-goettingen.de/schaback/teaching/sc.pdf 
+  [Item("MultiquadraticKernel", "A kernel function that uses the multi-quadratic function (sqrt(1+||x-c||²/β).")]
   public class MultiquadraticKernel : KernelBase {
 
@@ -41,5 +40,4 @@
 
     public MultiquadraticKernel() {
-      Parameters.Add(new FixedValueParameter<DoubleValue>(BetaParameterName, "The beta in the kernel function sqrt(1+||x-c||²/beta)", new DoubleValue(2)));
     }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -48,11 +46,13 @@
     #endregion
     protected override double Get(double norm) {
-      if (Math.Abs(Beta) < double.Epsilon) return double.NaN;
-      return Math.Sqrt(1 + norm * norm / Beta);
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      return Math.Sqrt(1 + norm * norm / beta);
     }
 
     protected override double GetGradient(double norm) {
-      if (Math.Abs(Beta) < double.Epsilon) return double.NaN;
-      var dividend = 2 * Beta * Beta * Math.Sqrt((Beta + norm * norm) / Beta);
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      var dividend = 2 * beta * beta * Math.Sqrt((beta + norm * norm) / beta);
       return -norm * norm / dividend;
     }
Index: branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/PolysplineKernel.cs
===================================================================
--- branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/PolysplineKernel.cs	(revision 14885)
+++ branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/PolysplineKernel.cs	(revision 14887)
@@ -22,12 +22,11 @@
 using System;
 using HeuristicLab.Common;
-using HeuristicLab.Core;
-using HeuristicLab.Data;
-using HeuristicLab.Parameters;
+using HeuristicLab.Core;         
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
-namespace HeuristicLab.Algorithms.DataAnalysis {
+namespace HeuristicLab.Algorithms.DataAnalysis.KernelRidgeRegression {
   [StorableClass]
-  [Item("PolysplineKernel", "A kernel function that uses the multiquadratic function")]
+  // conditionally positive definite. (need to add polynomials) see http://num.math.uni-goettingen.de/schaback/teaching/sc.pdf 
+  [Item("PolysplineKernel", "A kernel function that uses the poly-spline function ||x-c||^Beta.")]
   public class PolysplineKernel : KernelBase {
 
@@ -40,5 +39,4 @@
                 : base(original, cloner) { }
     public PolysplineKernel() {
-      Parameters.Add(new FixedValueParameter<DoubleValue>(BetaParameterName, "The Beta in the kernelfunction ||x-c||^Beta", new DoubleValue(1.5)));
     }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -48,9 +46,9 @@
 
     protected override double Get(double norm) {
-      return Math.Pow(norm, Beta);
+      return Math.Pow(norm, Beta.Value);
     }
 
     protected override double GetGradient(double norm) {
-      return Math.Pow(norm, Beta) * Math.Log(norm);
+      return Math.Pow(norm, Beta.Value) * Math.Log(norm);
     }
   }
Index: branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/ThinPlatePolysplineKernel.cs
===================================================================
--- branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/ThinPlatePolysplineKernel.cs	(revision 14885)
+++ branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/ThinPlatePolysplineKernel.cs	(revision 14887)
@@ -23,11 +23,10 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
-using HeuristicLab.Data;
-using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
-namespace HeuristicLab.Algorithms.DataAnalysis {
+namespace HeuristicLab.Algorithms.DataAnalysis.KernelRidgeRegression {
   [StorableClass]
-  [Item("ThinPlatePolysplineKernel", "A kernel function that uses the ThinPlatePolyspline function")]
+  // conditionally positive definite. (need to add polynomials) see http://num.math.uni-goettingen.de/schaback/teaching/sc.pdf 
+  [Item("ThinPlatePolysplineKernel", "A kernel function that uses the ThinPlatePolyspline function ||x-c||^(2*Beta)*log(||x-c||^Beta)")]
   public class ThinPlatePolysplineKernel : KernelBase {
     #region HLConstructors & Boilerplate
@@ -38,5 +37,4 @@
     protected ThinPlatePolysplineKernel(ThinPlatePolysplineKernel original, Cloner cloner) : base(original, cloner) { }
     public ThinPlatePolysplineKernel() {
-      Parameters.Add(new FixedValueParameter<DoubleValue>(BetaParameterName, "The Beta in the kernelfunction ||x-c||^(2*Beta)*log(||x-c||^Beta)", new DoubleValue(1)));
     }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -46,11 +44,13 @@
 
     protected override double Get(double norm) {
-      if (Math.Pow(norm, Beta) <= 0) return double.NaN;
-      return Math.Pow(norm, 2 * Beta) * Math.Log(1 + Math.Pow(norm, Beta));
+      var beta = Beta.Value;
+      if (Math.Pow(norm, beta) < 0) return double.NaN;
+      return Math.Pow(norm, 2 * beta) * Math.Log(1 + Math.Pow(norm, beta));
     }
 
     protected override double GetGradient(double norm) {
-      if (Math.Pow(norm, Beta) <= 0) return double.NaN;
-      return 2 * Math.Log(norm) * Math.Pow(norm, 2 * Beta) * Math.Log(1 + Math.Pow(norm, Beta)) + Math.Pow(norm, 3 * Beta) * Math.Log(norm) / (Math.Pow(norm, Beta) + 1);
+      var beta = Beta.Value;
+      if (Math.Pow(norm, beta) <= 0) return double.NaN;
+      return 2 * Math.Log(norm) * Math.Pow(norm, 2 * beta) * Math.Log(1 + Math.Pow(norm, beta)) + Math.Pow(norm, 3 * beta) * Math.Log(norm) / (Math.Pow(norm, beta) + 1);
     }
   }
Index: branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegression.cs
===================================================================
--- branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegression.cs	(revision 14885)
+++ branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegression.cs	(revision 14887)
@@ -31,13 +31,10 @@
 using HeuristicLab.Problems.DataAnalysis;
 
-namespace HeuristicLab.Algorithms.DataAnalysis {
-  /// <summary>
-  /// Linear regression data analysis algorithm.
-  /// </summary>
-  [Item("Radial Basis Function Regression (RBF-R)", "Radial basis function regression data analysis algorithm.")]
+namespace HeuristicLab.Algorithms.DataAnalysis.KernelRidgeRegression {
+  [Item("Kernel Ridge Regression", "Kernelized ridge regression e.g. for radial basis function (RBF) regression.")]
   [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 100)]
   [StorableClass]
-  public sealed class RadialBasisRegression : BasicAlgorithm {
-    private const string RBFRegressionSolutionResultName = "RBF regression solution";
+  public sealed class KernelRidgeRegression : BasicAlgorithm {
+    private const string SolutionResultName = "Kernel ridge regression solution";
 
     public override bool SupportsPause {
@@ -55,9 +52,11 @@
     private const string KernelParameterName = "Kernel";
     private const string ScaleInputVariablesParameterName = "ScaleInputVariables";
+    private const string LambdaParameterName = "LogLambda";
+    private const string BetaParameterName = "Beta";
     #endregion
 
     #region parameter properties
-    public ValueParameter<ICovarianceFunction> KernelParameter {
-      get { return (ValueParameter<ICovarianceFunction>)Parameters[KernelParameterName]; }
+    public ValueParameter<IKernel> KernelParameter {
+      get { return (ValueParameter<IKernel>)Parameters[KernelParameterName]; }
     }
 
@@ -65,8 +64,16 @@
       get { return (IFixedValueParameter<BoolValue>)Parameters[ScaleInputVariablesParameterName]; }
     }
+
+    public IFixedValueParameter<DoubleValue> LogLambdaParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[LambdaParameterName]; }
+    }
+
+    public IFixedValueParameter<DoubleValue> BetaParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[BetaParameterName]; }
+    }
     #endregion
 
     #region properties
-    public ICovarianceFunction Kernel {
+    public IKernel Kernel {
       get { return KernelParameter.Value; }
     }
@@ -77,17 +84,26 @@
     }
 
+    public double LogLambda {
+      get { return LogLambdaParameter.Value.Value; }
+      set { LogLambdaParameter.Value.Value = value; }
+    }
+
+    public double Beta {
+      get { return BetaParameter.Value.Value; }
+      set { BetaParameter.Value.Value = value; }
+    }
     #endregion
 
     [StorableConstructor]
-    private RadialBasisRegression(bool deserializing) : base(deserializing) { }
-    private RadialBasisRegression(RadialBasisRegression original, Cloner cloner)
+    private KernelRidgeRegression(bool deserializing) : base(deserializing) { }
+    private KernelRidgeRegression(KernelRidgeRegression original, Cloner cloner)
       : base(original, cloner) {
     }
-    public RadialBasisRegression() {
+    public KernelRidgeRegression() {
       Problem = new RegressionProblem();
-      Parameters.Add(new ValueParameter<ICovarianceFunction>(KernelParameterName, "The radial basis function"));
+      Parameters.Add(new ValueParameter<IKernel>(KernelParameterName, "The kernel", new GaussianKernel()));
       Parameters.Add(new FixedValueParameter<BoolValue>(ScaleInputVariablesParameterName, "Set to true if the input variables should be scaled to the interval [0..1]", new BoolValue(true)));
-      var kernel = new GaussianKernel();
-      KernelParameter.Value = kernel;
+      Parameters.Add(new FixedValueParameter<DoubleValue>(LambdaParameterName, "The log10-transformed weight for the regularization term lambda [-inf..+inf]. Small values produce more complex models, large values produce models with larger errors. Set to very small value (e.g. -1.0e15) for almost exact approximation", new DoubleValue(-2)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(BetaParameterName, "The beta parameter for the kernel", new DoubleValue(2)));
     }
     [StorableHook(HookType.AfterDeserialization)]
@@ -95,24 +111,27 @@
 
     public override IDeepCloneable Clone(Cloner cloner) {
-      return new RadialBasisRegression(this, cloner);
+      return new KernelRidgeRegression(this, cloner);
     }
 
     protected override void Run(CancellationToken cancellationToken) {
-      double loocvrmse, rmsError;
-      var solution = CreateRadialBasisRegressionSolution(Problem.ProblemData, Kernel, ScaleInputVariables, out loocvrmse, out rmsError);
-      Results.Add(new Result(RBFRegressionSolutionResultName, "The RBF regression solution.", solution));
-      Results.Add(new Result("LOOCVRMSE", "The root mean squared error of a leave-one-out-cross-validation on the training set", new DoubleValue(loocvrmse)));
+      double rmsError;
+      var kernel = Kernel;
+      kernel.Beta = Beta;
+      var solution = CreateRadialBasisRegressionSolution(Problem.ProblemData, kernel, Math.Pow(10, LogLambda), ScaleInputVariables, out rmsError);
+      Results.Add(new Result(SolutionResultName, "The kernel ridge regression solution.", solution));
       Results.Add(new Result("RMSE (test)", "The root mean squared error of the solution on the test set.", new DoubleValue(rmsError)));
     }
 
-    public static IRegressionSolution CreateRadialBasisRegressionSolution(IRegressionProblemData problemData, ICovarianceFunction kernel, bool scaleInputs, out double loocvRmsError, out double rmsError) {
-      var model = new RadialBasisFunctionModel(problemData.Dataset, problemData.TargetVariable, problemData.AllowedInputVariables, problemData.TrainingIndices, scaleInputs, kernel);
-      loocvRmsError = model.LeaveOneOutCrossValidationRootMeanSquaredError();
-      rmsError = Math.Sqrt(model.GetEstimatedValues(problemData.Dataset, problemData.TestIndices)
-        .Zip(problemData.TargetVariableTestValues, (a, b) => (a - b) * (a - b))
-        .Average());
+    public static IRegressionSolution CreateRadialBasisRegressionSolution(IRegressionProblemData problemData, ICovarianceFunction kernel, double lambda, bool scaleInputs, out double rmsError) {
+      var model = new KernelRidgeRegressionModel(problemData.Dataset, problemData.TargetVariable, problemData.AllowedInputVariables, problemData.TrainingIndices, scaleInputs, kernel, lambda);
+      rmsError = double.NaN;
+      if (problemData.TestIndices.Any()) {
+        rmsError = Math.Sqrt(model.GetEstimatedValues(problemData.Dataset, problemData.TestIndices)
+          .Zip(problemData.TargetVariableTestValues, (a, b) => (a - b) * (a - b))
+          .Average());
+      }
       var solution = model.CreateRegressionSolution((IRegressionProblemData)problemData.Clone());
-      solution.Model.Name = "RBF Regression Model";
-      solution.Name = "RBF Regression Solution";
+      solution.Model.Name = "Kernel ridge regression model";
+      solution.Name = SolutionResultName;
       return solution;
     }
Index: branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegressionModel.cs
===================================================================
--- branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegressionModel.cs	(revision 14885)
+++ branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegressionModel.cs	(revision 14887)
@@ -21,6 +21,5 @@
 
 using System;
-using System.Collections.Generic;
-using System.Diagnostics;
+using System.Collections.Generic;    
 using System.Linq;
 using HeuristicLab.Common;
@@ -29,11 +28,8 @@
 using HeuristicLab.Problems.DataAnalysis;
 
-namespace HeuristicLab.Algorithms.DataAnalysis {
-  /// <summary>
-  /// Represents an RBF regression model.
-  /// </summary>
+namespace HeuristicLab.Algorithms.DataAnalysis.KernelRidgeRegression {
   [StorableClass]
-  [Item("RBFModel", "An RBF regression model")]
-  public sealed class RadialBasisFunctionModel : RegressionModel, IConfidenceRegressionModel {
+  [Item("KernelRidgeRegressionModel", "A kernel ridge regression model")]
+  public sealed class KernelRidgeRegressionModel : RegressionModel {
     public override IEnumerable<string> VariablesUsedForPrediction {
       get { return allowedInputVariables; }
@@ -58,12 +54,16 @@
     private readonly ICovarianceFunction kernel;
 
-    private double[,] gramInvert; // not storable as it can be large (recreate after deserialization as required)
+    [Storable]
+    private readonly double lambda;
 
     [Storable]
-    private readonly double meanOffset; // implementation works for zero-mean target variables
+    private readonly double yOffset; // implementation works for zero-mean target variables
+
+    [Storable]
+    private readonly double yScale;
 
     [StorableConstructor]
-    private RadialBasisFunctionModel(bool deserializing) : base(deserializing) { }
-    private RadialBasisFunctionModel(RadialBasisFunctionModel original, Cloner cloner)
+    private KernelRidgeRegressionModel(bool deserializing) : base(deserializing) { }
+    private KernelRidgeRegressionModel(KernelRidgeRegressionModel original, Cloner cloner)
       : base(original, cloner) {
       // shallow copies of arrays because they cannot be modified
@@ -71,14 +71,18 @@
       alpha = original.alpha;
       trainX = original.trainX;
-      gramInvert = original.gramInvert;
       scaling = original.scaling;
+      lambda = original.lambda;
 
-      meanOffset = original.meanOffset;
+      yOffset = original.yOffset;
+      yScale = original.yScale;
       if (original.kernel != null)
         kernel = cloner.Clone(original.kernel);
     }
-    public RadialBasisFunctionModel(IDataset dataset, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<int> rows,
-      bool scaleInputs, ICovarianceFunction kernel)
-      : base(targetVariable) {
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new KernelRidgeRegressionModel(this, cloner);
+    }
+
+    public KernelRidgeRegressionModel(IDataset dataset, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<int> rows,
+      bool scaleInputs, ICovarianceFunction kernel, double lambda = 0.1) : base(targetVariable) {
       if (kernel.GetNumberOfParameters(allowedInputVariables.Count()) > 0) throw new ArgumentException("All parameters in the kernel function must be specified.");
       name = ItemName;
@@ -87,4 +91,5 @@
       var trainingRows = rows.ToArray();
       this.kernel = (ICovarianceFunction)kernel.Clone();
+      this.lambda = lambda;
       try {
         if (scaleInputs)
@@ -92,22 +97,63 @@
         trainX = ExtractData(dataset, trainingRows, scaling);
         var y = dataset.GetDoubleValues(targetVariable, trainingRows).ToArray();
-        meanOffset = y.Average();
-        for (int i = 0; i < y.Length; i++) y[i] -= meanOffset;
+        yOffset = y.Average();
+        yScale = 1.0 / y.StandardDeviation();
+        for (int i = 0; i < y.Length; i++) {
+          y[i] -= yOffset;
+          y[i] *= yScale;
+        }
         int info;
-        // TODO: improve efficiency by decomposing matrix once instead of solving the system and then inverting the matrix
-        alglib.densesolverlsreport denseSolveRep;
-        gramInvert = BuildGramMatrix(trainX);
+        alglib.densesolverreport denseSolveRep;
+        var gram = BuildGramMatrix(trainX, lambda); 
         int n = trainX.GetLength(0);
-        alglib.rmatrixsolvels(gramInvert, n, n, y, 0.0, out info, out denseSolveRep, out alpha);
+
+        // cholesky decomposition
+        var res = alglib.trfac.spdmatrixcholesky(ref gram, n, false);
+        if(res == false) throw new ArgumentException("Could not decompose matrix. Is it quadratic symmetric positive definite?");
+
+        alglib.spdmatrixcholeskysolve(gram, n, false, y, out info, out denseSolveRep, out alpha);
         if (info != 1) throw new ArgumentException("Could not create model.");
-
-        alglib.matinvreport report;
-        alglib.rmatrixinverse(ref gramInvert, out info, out report);
-        if (info != 1) throw new ArgumentException("Could not invert matrix. Is it quadratic symmetric positive definite?");
-
       } catch (alglib.alglibexception ae) {
         // wrap exception so that calling code doesn't have to know about alglib implementation
-        throw new ArgumentException("There was a problem in the calculation of the RBF model", ae);
+        throw new ArgumentException("There was a problem in the calculation of the kernel ridge regression model", ae);
       }
+    }
+
+
+    #region IRegressionModel Members
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      var newX = ExtractData(dataset, rows, scaling);
+      var dim = newX.GetLength(1);
+      var cov = kernel.GetParameterizedCovarianceFunction(new double[0], Enumerable.Range(0, dim).ToArray());
+
+      var pred = new double[newX.GetLength(0)];
+      for (int i = 0; i < pred.Length; i++) {
+        double sum = 0.0;
+        for (int j = 0; j < alpha.Length; j++) {
+          sum += alpha[j] * cov.CrossCovariance(trainX, newX, j, i);
+        }
+        pred[i] = sum / yScale + yOffset;
+      }
+      return pred;
+    }
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RegressionSolution(this, new RegressionProblemData(problemData));
+    }
+    #endregion
+
+    #region helpers
+    private double[,] BuildGramMatrix(double[,] data, double lambda) {
+      var n = data.GetLength(0);
+      var dim = data.GetLength(1);
+      var cov = kernel.GetParameterizedCovarianceFunction(new double[0], Enumerable.Range(0, dim).ToArray());
+      var gram = new double[n, n];
+      // G = (K + λ I) 
+      for (var i = 0; i < n; i++) {
+        for (var j = i; j < n; j++) {
+          gram[j, i] = cov.Covariance(data, i, j); // symmetric matrix --> fill only lower triangle
+        }
+        gram[i, i] += lambda;
+      }
+      return gram;
     }
 
@@ -145,93 +191,4 @@
       return res;
     }
-
-    public override IDeepCloneable Clone(Cloner cloner) {
-      return new RadialBasisFunctionModel(this, cloner);
-    }
-
-    #region IRegressionModel Members
-    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
-      var newX = ExtractData(dataset, rows, scaling);
-      var dim = newX.GetLength(1);
-      var cov = kernel.GetParameterizedCovarianceFunction(new double[0], Enumerable.Range(0, dim).ToArray());
-
-      var pred = new double[newX.GetLength(0)];
-      for (int i = 0; i < pred.Length; i++) {
-        double sum = meanOffset;
-        for (int j = 0; j < alpha.Length; j++) {
-          sum += alpha[j] * cov.CrossCovariance(trainX, newX, j, i);
-        }
-        pred[i] = sum;
-      }
-      return pred;
-    }
-    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
-      return new ConfidenceRegressionSolution(this, new RegressionProblemData(problemData));
-    }
-    #endregion
-
-    public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
-      if (gramInvert == null) CreateAndInvertGramMatrix();
-      int n = gramInvert.GetLength(0);
-      var newData = ExtractData(dataset, rows, scaling);
-      var dim = newData.GetLength(1);
-      var cov = kernel.GetParameterizedCovarianceFunction(new double[0], Enumerable.Range(0, dim).ToArray());
-
-      // TODO perf (matrix matrix multiplication)
-      for (int i = 0; i < newData.GetLength(0); i++) {
-        double[] p = new double[n];
-
-        for (int j = 0; j < trainX.GetLength(0); j++) {
-          p[j] = cov.CrossCovariance(trainX, newData, j, i);
-        }
-
-        var Ap = new double[n];
-        alglib.ablas.rmatrixmv(n, n, gramInvert, 0, 0, 0, p, 0, ref Ap, 0);
-        var res = 0.0;
-        // dot product
-        for (int j = 0; j < p.Length; j++) res += p[j] * Ap[j];
-        yield return res > 0 ? res : 0;
-      }
-    }
-    public double LeaveOneOutCrossValidationRootMeanSquaredError() {
-      if (gramInvert == null) CreateAndInvertGramMatrix();
-      var n = gramInvert.GetLength(0);
-      var s = 1.0 / n;
-
-      var sum = 0.0;
-      for (int i = 0; i < alpha.Length; i++) {
-        var x = alpha[i] / gramInvert[i, i];
-        sum += x * x;
-      }
-      sum *= s;
-      return Math.Sqrt(sum);
-    }
-
-    private void CreateAndInvertGramMatrix() {
-      try {
-        gramInvert = BuildGramMatrix(trainX);
-        int info = 0;
-        alglib.matinvreport report;
-        alglib.rmatrixinverse(ref gramInvert, out info, out report);
-        if (info != 1)
-          throw new ArgumentException("Could not invert matrix. Is it quadratic symmetric positive definite?");
-      } catch (alglib.alglibexception) {
-        // wrap exception so that calling code doesn't have to know about alglib implementation
-        throw new ArgumentException("Could not invert matrix. Is it quadratic symmetric positive definite?");
-      }
-    }
-    #region helpers
-    private double[,] BuildGramMatrix(double[,] data) {
-      var n = data.GetLength(0);
-      var dim = data.GetLength(1);
-      var cov = kernel.GetParameterizedCovarianceFunction(new double[0], Enumerable.Range(0, dim).ToArray());
-      var gram = new double[n, n];
-      for (var i = 0; i < n; i++)
-        for (var j = i; j < n; j++) {
-          gram[j, i] = gram[i, j] = cov.Covariance(data, i, j); // symmetric matrix --> half of the work
-        }
-      return gram;
-    }
-
     #endregion
   }
