Changeset 14891 for branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions

Timestamp:

04/26/17 11:06:51 (8 years ago)

Author:

bwerth

Message:

#2699 reworked kenel functions (beta is always a scaling factor now), added LU-Decomposition as a fall-back if Cholesky-decomposition fails

Location:

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions

Files:

• CicularKernel.cs (modified) (2 diffs)
• GaussianKernel.cs (modified) (2 diffs)
• InverseMultiquadraticKernel.cs (modified) (3 diffs)
• MultiquadraticKernel.cs (modified) (3 diffs)
• PolysplineKernel.cs (modified) (4 diffs)
• ThinPlatePolysplineKernel.cs (modified) (4 diffs)

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/CicularKernel.cs

@@ -27 +27 @@
 namespace HeuristicLab.Algorithms.DataAnalysis.KernelRidgeRegression {
   [StorableClass]
-  [Item("CircularKernel", "A circular kernel function 2*pi*(acos(-d)-d*(1-n²)^(0.5)) where n = ||x-c|| and d = n/beta")]
+  [Item("CircularKernel", "A circular kernel function 2*pi*(acos(-d)-d*(1-d²)^(0.5)) where n = ||x-c|| and d = n/beta \n  As described in http://crsouza.com/2010/03/17/kernel-functions-for-machine-learning-applications/")]
   public class CircularKernel : KernelBase {
 
@@ -45 +45 @@
     protected override double Get(double norm) {
       var beta = Beta.Value;
-      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
-      if (norm >= beta) return 0;
+      if (Math.Abs(Beta.Value) < double.Epsilon) return double.NaN;
+      if (norm >= Beta.Value) return 0;
       var d = norm / beta;
-      return Math.Acos(-d) - d * Math.Sqrt(1 - d * d) - Math.PI / 2;
+      return 2 * Math.PI * (Math.Acos(-d) - d * Math.Sqrt(1 - d * d));
     }
 
+    // 4*pi*n^3 / (beta^4 * sqrt(1-n^2/beta^2) 
     protected override double GetGradient(double norm) {
       var beta = Beta.Value;
       if (Math.Abs(beta) < double.Epsilon) return double.NaN;
       if (beta < norm) return 0;
-      return -2 * Math.Pow(norm, 3) / (Math.Pow(beta, 4) * Math.Sqrt(1 - norm * norm / (beta * beta)));
+      var d = norm / beta;
+      return -4 * Math.PI * d * d * d / beta * Math.Sqrt(1 - d * d);
     }
   }

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/GaussianKernel.cs

@@ -29 +29 @@
 namespace HeuristicLab.Algorithms.DataAnalysis.KernelRidgeRegression {
   [StorableClass]
-  [Item("GaussianKernel", "A kernel function that uses Gaussian function exp(-||x-c||/beta²). Positive definite beta > 0")]
+  [Item("GaussianKernel", "A kernel function that uses Gaussian function exp(-n²/beta²). As described in http://crsouza.com/2010/03/17/kernel-functions-for-machine-learning-applications/")]
   public class GaussianKernel : KernelBase {
 
@@ -48 +48 @@
       var beta = Beta.Value;
       if (Math.Abs(beta) < double.Epsilon) return double.NaN;
-      return Math.Exp(-norm * norm / (beta * beta));
+      var d = norm / beta;
+      return Math.Exp(-d * d);
     }
 
+    //2 * n²/b²* 1/b * exp(-n²/b²)
     protected override double GetGradient(double norm) {
       var beta = Beta.Value;
       if (Math.Abs(beta) < double.Epsilon) return double.NaN;
-      return 2 * norm * norm / Math.Pow(beta, 3) * Math.Exp(-norm * norm / (beta * beta));
+      var d = norm / beta;
+      return 2 * d * d / beta * Math.Exp(-d * d);
     }
   }

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/InverseMultiquadraticKernel.cs

@@ -22 +22 @@
 using System;
 using HeuristicLab.Common;
-using HeuristicLab.Core;     
+using HeuristicLab.Core;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.Algorithms.DataAnalysis.KernelRidgeRegression {
   [StorableClass]
-  [Item("InverseMultiquadraticKernel", "A kernel function that uses the inverse multi-quadratic function  1 / sqrt(1+||x-c||^2/beta). Positive definite: beta > 0")]
+  [Item("InverseMultiquadraticKernel", "A kernel function that uses the inverse multi-quadratic function  1 / sqrt(1+||x-c||²/beta²). Similar to http://crsouza.com/2010/03/17/kernel-functions-for-machine-learning-applications/ with beta as a scaling factor.")]
   public class InverseMultiquadraticKernel : KernelBase {
+
+    private const double C = 1.0;
     #region HLConstructors & Boilerplate
     [StorableConstructor]
@@ -35 +37 @@
     private void AfterDeserialization() { }
     protected InverseMultiquadraticKernel(InverseMultiquadraticKernel original, Cloner cloner) : base(original, cloner) { }
-    public InverseMultiquadraticKernel() {
-    }
+    public InverseMultiquadraticKernel() { }
     public override IDeepCloneable Clone(Cloner cloner) {
       return new InverseMultiquadraticKernel(this, cloner);
@@ -45 +46 @@
       var beta = Beta.Value;
       if (Math.Abs(beta) < double.Epsilon) return double.NaN;
-      return 1 / Math.Sqrt(1 + norm * norm / beta);
+      var d = norm / beta;
+      return 1 / Math.Sqrt(C + d * d);
     }
 
+    //n²/(b³(n²/b² + C)^1.5)
     protected override double GetGradient(double norm) {
       var beta = Beta.Value;
       if (Math.Abs(beta) < double.Epsilon) return double.NaN;
-      return norm * norm / (2 * beta * beta * Math.Pow((norm * norm + beta) / beta, 1.5));
+      var d = norm / beta;
+      return d * d / (beta * Math.Pow(d * d + C, 1.5));
     }
   }

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/MultiquadraticKernel.cs

@@ -22 +22 @@
 using System;
 using HeuristicLab.Common;
-using HeuristicLab.Core;            
+using HeuristicLab.Core;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
@@ -28 +28 @@
   [StorableClass]
   // conditionally positive definite. (need to add polynomials) see http://num.math.uni-goettingen.de/schaback/teaching/sc.pdf 
-  [Item("MultiquadraticKernel", "A kernel function that uses the multi-quadratic function (sqrt(1+||x-c||²/β).")]
+  [Item("MultiquadraticKernel", "A kernel function that uses the multi-quadratic function sqrt(1+||x-c||²/beta²). Similar to http://crsouza.com/2010/03/17/kernel-functions-for-machine-learning-applications/ with beta as a scaling factor.")]
   public class MultiquadraticKernel : KernelBase {
 
+    private const double C = 1.0;
     #region HLConstructors & Boilerplate
     [StorableConstructor]
@@ -48 +49 @@
       var beta = Beta.Value;
       if (Math.Abs(beta) < double.Epsilon) return double.NaN;
-      return Math.Sqrt(1 + norm * norm / beta);
+      var d = norm / beta;
+      return Math.Sqrt(C + d * d);
     }
 
+    //-n²/(d³*sqrt(C+n²/d²))
     protected override double GetGradient(double norm) {
       var beta = Beta.Value;
       if (Math.Abs(beta) < double.Epsilon) return double.NaN;
-      var dividend = 2 * beta * beta * Math.Sqrt((beta + norm * norm) / beta);
-      return -norm * norm / dividend;
+      var d = norm / beta;
+      return -d * d / (beta * Math.Sqrt(C + d * d));
     }
   }

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/PolysplineKernel.cs

@@ -22 +22 @@
 using System;
 using HeuristicLab.Common;
-using HeuristicLab.Core;         
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
@@ -28 +30 @@
   [StorableClass]
   // conditionally positive definite. (need to add polynomials) see http://num.math.uni-goettingen.de/schaback/teaching/sc.pdf 
-  [Item("PolysplineKernel", "A kernel function that uses the poly-spline function ||x-c||^Beta.")]
+  [Item("PolysplineKernel", "A kernel function that uses the polyharmonic function (||x-c||/Beta)^Degree as given in http://num.math.uni-goettingen.de/schaback/teaching/sc.pdf with beta as a scaling parameters.")]
   public class PolysplineKernel : KernelBase {
+
+    #region Parameternames
+    private const string DegreeParameterName = "Degree";
+    #endregion
+    #region Parameterproperties
+    public IFixedValueParameter<DoubleValue> DegreeParameter
+    {
+      get { return Parameters[DegreeParameterName] as IFixedValueParameter<DoubleValue>; }
+    }
+    #endregion
+    #region Properties
+    public DoubleValue Degree
+    {
+      get { return DegreeParameter.Value; }
+    }
+    #endregion
 
     #region HLConstructors & Boilerplate
@@ -36 +54 @@
     [StorableHook(HookType.AfterDeserialization)]
     private void AfterDeserialization() { }
-    protected PolysplineKernel(PolysplineKernel original, Cloner cloner)
-                : base(original, cloner) { }
+    protected PolysplineKernel(PolysplineKernel original, Cloner cloner) : base(original, cloner) { }
     public PolysplineKernel() {
+      Parameters.Add(new FixedValueParameter<DoubleValue>(DegreeParameterName, "The degree of the kernel. Needs to be greater than zero.", new DoubleValue(1.0)));
     }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -46 +64 @@
 
     protected override double Get(double norm) {
-      return Math.Pow(norm, Beta.Value);
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      var d = norm / beta;
+      return Math.Pow(d, Degree.Value);
     }
 
+    //-degree/beta * (norm/beta)^degree
     protected override double GetGradient(double norm) {
-      return Math.Pow(norm, Beta.Value) * Math.Log(norm);
+      var beta = Beta.Value;
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      var d = norm / beta;
+      return -Degree.Value / beta * Math.Pow(d, Degree.Value);
     }
   }

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelFunctions/ThinPlatePolysplineKernel.cs

@@ -23 +23 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
@@ -28 +30 @@
   [StorableClass]
   // conditionally positive definite. (need to add polynomials) see http://num.math.uni-goettingen.de/schaback/teaching/sc.pdf 
-  [Item("ThinPlatePolysplineKernel", "A kernel function that uses the ThinPlatePolyspline function ||x-c||^(2*Beta)*log(||x-c||^Beta)")]
+  [Item("ThinPlatePolysplineKernel", "A kernel function that uses the ThinPlatePolyspline function (||x-c||/Beta)^(Degree)*log(||x-c||/Beta) as described in \"Thin-Plate Spline Radial Basis Function Scheme for Advection-Diffusion Problems\" with beta as a scaling parameter.")]
   public class ThinPlatePolysplineKernel : KernelBase {
+
+    #region Parameternames
+    private const string DegreeParameterName = "Degree";
+    #endregion
+    #region Parameterproperties
+    public IFixedValueParameter<DoubleValue> DegreeParameter
+    {
+      get { return Parameters[DegreeParameterName] as IFixedValueParameter<DoubleValue>; }
+    }
+    #endregion
+    #region Properties
+    public DoubleValue Degree
+    {
+      get { return DegreeParameter.Value; }
+    }
+    #endregion
+
     #region HLConstructors & Boilerplate
     [StorableConstructor]
@@ -37 +56 @@
     protected ThinPlatePolysplineKernel(ThinPlatePolysplineKernel original, Cloner cloner) : base(original, cloner) { }
     public ThinPlatePolysplineKernel() {
+      Parameters.Add(new FixedValueParameter<DoubleValue>(DegreeParameterName, "The degree of the kernel. Needs to be greater than zero.", new DoubleValue(2.0)));
     }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -45 +65 @@
     protected override double Get(double norm) {
       var beta = Beta.Value;
-      if (Math.Pow(norm, beta) < 0) return double.NaN;
-      return Math.Pow(norm, 2 * beta) * Math.Log(1 + Math.Pow(norm, beta));
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      var d = norm / beta;
+      if (Math.Abs(d) < double.Epsilon) return 0;
+      return Math.Pow(d, Degree.Value) * Math.Log(d);
     }
 
+    // (Degree/beta) * (norm/beta)^Degree * log(norm/beta) 
     protected override double GetGradient(double norm) {
       var beta = Beta.Value;
-      if (Math.Pow(norm, beta) <= 0) return double.NaN;
-      return 2 * Math.Log(norm) * Math.Pow(norm, 2 * beta) * Math.Log(1 + Math.Pow(norm, beta)) + Math.Pow(norm, 3 * beta) * Math.Log(norm) / (Math.Pow(norm, beta) + 1);
+      if (Math.Abs(beta) < double.Epsilon) return double.NaN;
+      var d = norm / beta;
+      if (Math.Abs(d) < double.Epsilon) return 0;
+      return Degree.Value / beta * Math.Pow(d, Degree.Value) * Math.Log(d);
     }
   }