Changeset 8473

Timestamp:

08/10/12 20:42:30 (12 years ago)

Author:

gkronber

Message:

#1902 worked on GPR: added line chart, made parameters of mean and covariance functions readable, removed target variable scaling, moved noise hyperparameter for likelihood function to the end of the parameter list, added methods to calculate the predicted variance, removed limits for scale of covariance functions and introduced exception handling to catch non-spd or singular cov matrixes, implemented rational quadratic covariance function, added unit test case from GBML book (however it does not work as the book seemingly uses a noise-less likelihood function)

Location:

trunk/sources

Files:

• HeuristicLab.Algorithms.DataAnalysis.Views/3.4/GaussianProcessRegressionSolutionLineChartView.Designer.cs (added)
• HeuristicLab.Algorithms.DataAnalysis.Views/3.4/GaussianProcessRegressionSolutionLineChartView.cs (added)
• HeuristicLab.Algorithms.DataAnalysis.Views/3.4/HeuristicLab.Algorithms.DataAnalysis.Views-3.4.csproj (modified) (3 diffs)
• HeuristicLab.Algorithms.DataAnalysis.Views/3.4/Plugin.cs.frame (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceConst.cs (modified) (2 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceNoise.cs (modified) (2 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovariancePeriodic.cs (modified) (2 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceRQiso.cs (added)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceSEard.cs (modified) (2 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceSEiso.cs (modified) (2 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs (modified) (10 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs (modified) (2 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolution.cs (modified) (2 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanConst.cs (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanLinear.cs (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModel.cs (modified) (2 diffs)
• HeuristicLab.Algorithms.GradientDescent/3.3/LbfgsInitializer.cs (modified) (1 diff)
• HeuristicLab.Tests/HeuristicLab.Algorithms.DataAnalysis-3.4/GaussianProcessFunctionsTest.cs (modified) (6 diffs)
• HeuristicLab.Tests/HeuristicLab.Algorithms.DataAnalysis-3.4/GaussianProcessModelTest.cs (modified) (3 diffs)

trunk/sources/HeuristicLab.Algorithms.DataAnalysis.Views/3.4/HeuristicLab.Algorithms.DataAnalysis.Views-3.4.csproj

@@ -112 +112 @@
     <Reference Include="System.Drawing" />
     <Reference Include="System.Windows.Forms" />
+    <Reference Include="System.Windows.Forms.DataVisualization" />
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />
@@ -161 +162 @@
     </Compile>
     <Compile Include="Plugin.cs" />
+    <Compile Include="GaussianProcessRegressionSolutionLineChartView.cs">
+      <SubType>UserControl</SubType>
+    </Compile>
+    <Compile Include="GaussianProcessRegressionSolutionLineChartView.Designer.cs">
+      <DependentUpon>GaussianProcessRegressionSolutionLineChartView.cs</DependentUpon>
+    </Compile>
     <Compile Include="SupportVectorMachineModelSupportVectorsView.cs">
       <SubType>UserControl</SubType>
@@ -265 +272 @@
       <Project>{DF87C13E-A889-46FF-8153-66DCAA8C5674}</Project>
       <Name>HeuristicLab.Problems.DataAnalysis-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Visualization.ChartControlsExtensions\3.3\HeuristicLab.Visualization.ChartControlsExtensions-3.3.csproj">
+      <Project>{315BDA09-3F4F-49B3-9790-B37CFC1C5750}</Project>
+      <Name>HeuristicLab.Visualization.ChartControlsExtensions-3.3</Name>
       <Private>False</Private>
     </ProjectReference>

trunk/sources/HeuristicLab.Algorithms.DataAnalysis.Views/3.4/Plugin.cs.frame

@@ -44 +44 @@
   [PluginDependency("HeuristicLab.Problems.DataAnalysis", "3.4")]
   [PluginDependency("HeuristicLab.Problems.DataAnalysis.Views", "3.4")]
+  [PluginDependency("HeuristicLab.Visualization.ChartControlsExtensions", "3.3")]
   public class HeuristicLabAlgorithmsDataAnalysisViewsPlugin : PluginBase {
   }

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceConst.cs

@@ -32 +32 @@
     [Storable]
     private double sf2;
+    public double Scale { get { return sf2; } }
 
     [StorableConstructor]
@@ -56 +57 @@
 
     public void SetParameter(double[] hyp) {
-      this.sf2 = Math.Min(1E6, Math.Exp(2 * hyp[0])); // upper limit for scale
+      this.sf2 = Math.Exp(2 * hyp[0]);
     }
     public void SetData(double[,] x) {

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceNoise.cs

@@ -32 +32 @@
     [Storable]
     private double sf2;
+    public double Scale { get { return sf2; } }
 
     [StorableConstructor]
@@ -56 +57 @@
 
     public void SetParameter(double[] hyp) {
-      this.sf2 = Math.Min(1E6, Math.Exp(2 * hyp[0])); // upper limit for scale
+      this.sf2 = Math.Exp(2 * hyp[0]);
     }
     public void SetData(double[,] x) {

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovariancePeriodic.cs

@@ -35 +35 @@
     [Storable]
     private double sf2;
+    public double Scale { get { return sf2; } }
     [Storable]
     private double l;
+    public double Length { get { return l; } }
     [Storable]
     private double p;
+    public double Period { get { return p; } }
 
     private bool symmetric;
@@ -74 +77 @@
       this.p = Math.Exp(hyp[1]);
       this.sf2 = Math.Exp(2 * hyp[2]);
-
-      sf2 = Math.Min(10E6, sf2); // upper limit for the scale
+      // sf2 = Math.Min(10E6, sf2); // upper limit for the scale
 
       sd = null;

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceSEard.cs

@@ -36 +36 @@
     [Storable]
     private double sf2;
+    public double Scale { get { return sf2; } }
+
     [Storable]
     private double[] l;
+    public double[] Length {
+      get {
+        if (l == null) return new double[0];
+        var copy = new double[l.Length];
+        Array.Copy(l, copy, copy.Length);
+        return copy;
+      }
+    }
 
     private double[,] sd;
@@ -76 +86 @@
       this.l = hyp.Take(hyp.Length - 1).Select(Math.Exp).ToArray();
       this.sf2 = Math.Exp(2 * hyp[hyp.Length - 1]);
-      sf2 = Math.Min(10E6, sf2); // upper limit for the scale
+      // sf2 = Math.Min(10E6, sf2); // upper limit for the scale
 
       sd = null;

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceSEiso.cs

@@ -37 +37 @@
     [Storable]
     private double sf2;
+    public double Scale { get { return sf2; } }
     [Storable]
     private double l;
+    public double Length { get { return l; } }
     [Storable]
     private bool symmetric;
@@ -80 +82 @@
     public void SetParameter(double[] hyp) {
       this.l = Math.Exp(hyp[0]);
-      this.sf2 = Math.Min(1E6, Math.Exp(2 * hyp[1])); // upper limit for scale
+      this.sf2 = Math.Exp(2 * hyp[1]);
       sd = null;
     }

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs

@@ -74 +74 @@
     [Storable]
     private Scaling inputScaling;
-    [Storable]
-    private Scaling targetScaling;
 
 
@@ -85 +83 @@
       this.covarianceFunction = cloner.Clone(original.covarianceFunction);
       this.inputScaling = cloner.Clone(original.inputScaling);
-      this.targetScaling = cloner.Clone(original.targetScaling);
       this.negativeLogLikelihood = original.negativeLogLikelihood;
       this.targetVariable = original.targetVariable;
@@ -106 +103 @@
       this.allowedInputVariables = allowedInputVariables.ToArray();
 
-      sqrSigmaNoise = Math.Exp(2.0 * hyp.First());
-      sqrSigmaNoise = Math.Max(10E-6, sqrSigmaNoise); // lower limit for the noise level
 
       int nVariables = this.allowedInputVariables.Length;
-      this.meanFunction.SetParameter(hyp.Skip(1)
+      this.meanFunction.SetParameter(hyp
         .Take(this.meanFunction.GetNumberOfParameters(nVariables))
         .ToArray());
-      this.covarianceFunction.SetParameter(hyp.Skip(1 + this.meanFunction.GetNumberOfParameters(nVariables))
+      this.covarianceFunction.SetParameter(hyp.Skip(this.meanFunction.GetNumberOfParameters(nVariables))
         .Take(this.covarianceFunction.GetNumberOfParameters(nVariables))
         .ToArray());
+      sqrSigmaNoise = Math.Exp(2.0 * hyp.Last());
 
       CalculateModel(ds, rows);
@@ -123 +119 @@
       inputScaling = new Scaling(ds, allowedInputVariables, rows);
       x = AlglibUtil.PrepareAndScaleInputMatrix(ds, allowedInputVariables, rows, inputScaling);
-
-
-      targetScaling = new Scaling(ds, new string[] { targetVariable }, rows);
-      var y = targetScaling.GetScaledValues(ds, targetVariable, rows);
+      var y = ds.GetDoubleValues(targetVariable, rows);
 
       int n = x.GetLength(0);
@@ -149 +142 @@
 
       var res = alglib.trfac.spdmatrixcholesky(ref l, n, false);
-      if (!res) throw new InvalidOperationException("Matrix is not positive semidefinite");
+      if (!res) throw new ArgumentException("Matrix is not positive semidefinite");
 
       // calculate sum of diagonal elements for likelihood
@@ -198 +191 @@
       }
 
-      return new double[] { noiseGradient }
-        .Concat(meanGradients)
-        .Concat(covGradients).ToArray();
+      return
+        meanGradients
+        .Concat(covGradients)
+        .Concat(new double[] { noiseGradient }).ToArray();
     }
 
@@ -231 +225 @@
       // var kss = new double[newN];
       var Ks = new double[newN, n];
-      double[,] sWKs = new double[n, newN];
+      //double[,] sWKs = new double[n, newN];
       // double[,] v;
 
@@ -243 +237 @@
       var ms = meanFunction.GetMean(newX);
       for (int i = 0; i < newN; i++) {
-
         for (int j = 0; j < n; j++) {
           Ks[i, j] = covarianceFunction.GetCovariance(j, i);
-          sWKs[j, i] = Ks[i, j] / Math.Sqrt(sqrSigmaNoise);
+          //sWKs[j, i] = Ks[i, j] / Math.Sqrt(sqrSigmaNoise);
         }
       }
@@ -253 +246 @@
       // alglib.rmatrixsolvem(l, n, sWKs, newN, true, out info, out denseSolveRep, out v);
 
-      double targetScaleMin, targetScaleMax;
-      targetScaling.GetScalingParameters(targetVariable, out targetScaleMin, out targetScaleMax);
       return Enumerable.Range(0, newN)
-        .Select(i => ms[i] + Util.ScalarProd(Util.GetRow(Ks, i), alpha))
-        .Select(m => m * (targetScaleMax - targetScaleMin) + targetScaleMin);
+        .Select(i => ms[i] + Util.ScalarProd(Util.GetRow(Ks, i), alpha));
       //for (int i = 0; i < newN; i++) {
       //  // predMean[i] = ms[i] + prod(GetRow(Ks, i), alpha);
@@ -265 +255 @@
 
     }
+
+    public IEnumerable<double> GetEstimatedVariance(Dataset dataset, IEnumerable<int> rows) {
+      var newX = AlglibUtil.PrepareAndScaleInputMatrix(dataset, allowedInputVariables, rows, inputScaling);
+      int newN = newX.GetLength(0);
+      int n = x.GetLength(0);
+
+      var kss = new double[newN];
+      double[,] sWKs = new double[n, newN];
+
+
+      // for stddev 
+      covarianceFunction.SetData(newX);
+      for (int i = 0; i < newN; i++)
+        kss[i] = covarianceFunction.GetCovariance(i, i);
+
+      covarianceFunction.SetData(x, newX);
+      for (int i = 0; i < n; i++) {
+        for (int j = 0; j < newN; j++) {
+          sWKs[i, j] = covarianceFunction.GetCovariance(i, j) / Math.Sqrt(sqrSigmaNoise);
+        }
+      }
+
+      // for stddev 
+      int info;
+      alglib.densesolverreport denseSolveRep;
+      double[,] v;
+      double[,] lTrans = new double[l.GetLength(1), l.GetLength(0)];
+      for (int i = 0; i < lTrans.GetLength(0); i++)
+        for (int j = 0; j < lTrans.GetLength(1); j++)
+          lTrans[i, j] = l[j, i];
+      alglib.rmatrixsolvem(lTrans, n, sWKs, newN, true, out info, out denseSolveRep, out v); // not working!
+      // alglib.spdmatrixcholeskysolvem(lTrans, n, true, sWKs, newN, out info, out denseSolveRep, out v);
+
+      for (int i = 0; i < newN; i++) {
+        var sumV2 = Util.ScalarProd(Util.GetCol(v, i), Util.GetCol(v, i));
+        yield return kss[i] - sumV2;
+      }
+    }
   }
 }

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs

@@ -20 +20 @@
 #endregion
 
+using System;
 using System.Linq;
 using HeuristicLab.Common;
@@ -60 +61 @@
 
     public override IOperation Apply() {
-      var model = Create(ProblemData, Hyperparameter.ToArray(), MeanFunction, CovarianceFunction);
-      ModelParameter.ActualValue = model;
-      NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(model.NegativeLogLikelihood);
-      HyperparameterGradientsParameter.ActualValue = new RealVector(model.GetHyperparameterGradients());
+      try {
+        var model = Create(ProblemData, Hyperparameter.ToArray(), MeanFunction, CovarianceFunction);
+        ModelParameter.ActualValue = model;
+        NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(model.NegativeLogLikelihood);
+        HyperparameterGradientsParameter.ActualValue = new RealVector(model.GetHyperparameterGradients());
+        return base.Apply();
+      }
+      catch (ArgumentException) { }
+      catch (alglib.alglibexception) { }
+      NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(1E300);
+      HyperparameterGradientsParameter.ActualValue = new RealVector(Hyperparameter.Count());
       return base.Apply();
     }

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolution.cs

@@ -20 +20 @@
 #endregion
 
+using System.Collections.Generic;
+using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -51 +53 @@
       return new GaussianProcessRegressionSolution(this, cloner);
     }
+
+    public IEnumerable<double> EstimatedVariance {
+      get { return GetEstimatedVariance(Enumerable.Range(0, ProblemData.Dataset.Rows)); }
+    }
+    public IEnumerable<double> EstimatedTrainingVariance {
+      get { return GetEstimatedVariance(ProblemData.TrainingIndices); }
+    }
+    public IEnumerable<double> EstimatedTestVariance {
+      get { return GetEstimatedVariance(ProblemData.TestIndices); }
+    }
+
+    public IEnumerable<double> GetEstimatedVariance(IEnumerable<int> rows) {
+      return Model.GetEstimatedVariance(ProblemData.Dataset, rows);
+    }
   }
 }

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanConst.cs

@@ -32 +32 @@
     [Storable]
     private double c;
+    public double Value { get { return c; } }
+
     public int GetNumberOfParameters(int numberOfVariables) {
       return 1;

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanLinear.cs

@@ -31 +31 @@
     [Storable]
     private double[] alpha;
+    public double[] Weights {
+      get {
+        if (alpha == null) return new double[0];
+        var copy = new double[alpha.Length];
+        Array.Copy(alpha, copy, copy.Length);
+        return copy;
+      }
+    }
     public int GetNumberOfParameters(int numberOfVariables) {
       return numberOfVariables;

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj

@@ -120 +120 @@
     </Compile>
     <Compile Include="FixedDataAnalysisAlgorithm.cs" />
+    <Compile Include="GaussianProcess\CovarianceRQiso.cs" />
     <Compile Include="GaussianProcess\CovarianceNoise.cs" />
     <Compile Include="GaussianProcess\CovarianceConst.cs" />

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModel.cs

@@ -20 +20 @@
 #endregion
 
+using System.Collections.Generic;
 using HeuristicLab.Problems.DataAnalysis;
 
@@ -31 +32 @@
     ICovarianceFunction CovarianceFunction { get; }
     double[] GetHyperparameterGradients();
+
+    IEnumerable<double> GetEstimatedVariance(Dataset ds, IEnumerable<int> rows);
   }
 }

trunk/sources/HeuristicLab.Algorithms.GradientDescent/3.3/LbfgsInitializer.cs

@@ -87 +87 @@
       alglib.minlbfgs.minlbfgsstate state = new alglib.minlbfgs.minlbfgsstate();
       if (ApproximateGradients.Value) {
-        alglib.minlbfgs.minlbfgscreatef(n, Math.Min(n, 7), initialPoint, 1E-5, state);
+        alglib.minlbfgs.minlbfgscreatef(n, Math.Min(n, 10), initialPoint, 1E-5, state);
       } else {
-        alglib.minlbfgs.minlbfgscreate(n, Math.Min(n, 7), initialPoint, state);
+        alglib.minlbfgs.minlbfgscreate(n, Math.Min(n, 10), initialPoint, state);
       }
-      alglib.minlbfgs.minlbfgssetcond(state, 0, 0, 0, Iterations.Value);
+      alglib.minlbfgs.minlbfgssetcond(state, 0.0, 0, 0, Iterations.Value);
       alglib.minlbfgs.minlbfgssetxrep(state, true);
 

trunk/sources/HeuristicLab.Tests/HeuristicLab.Algorithms.DataAnalysis-3.4/GaussianProcessFunctionsTest.cs

@@ -254 +254 @@
     }
 
+    [TestMethod]
+    public void CovRQIsoTest() {
+      TestCovarianceFunction(new CovarianceRQiso(), 0,
+        new double[,]
+          {
+{    0.6452,    0.6190,    0.8662,    0.6338,    0.7802,    0.5893,    0.7102,    0.7753,    0.7037,    0.8331},
+{    0.6771,    0.7392,    0.7442,    0.6630,    0.7603,    0.6144,    0.7145,    0.8346,    0.7492,    0.8200},
+{    0.7963,    0.7777,    0.8574,    0.7266,    0.6362,    0.8060,    0.8519,    0.7084,    0.7810,    0.8459},
+{    0.7988,    0.6268,    0.8004,    0.7500,    0.5583,    0.6524,    0.8968,    0.8864,    0.8739,    0.8382},
+{    0.8902,    0.6571,    0.7919,    0.9030,    0.5304,    0.5727,    0.8079,    0.9232,    0.9117,    0.8733},
+{    0.8819,    0.7639,    0.7387,    0.8449,    0.5785,    0.6563,    0.8783,    0.9519,    0.9535,    0.8742},
+{    0.6716,    0.7814,    0.7590,    0.6437,    0.8797,    0.6366,    0.7012,    0.6810,    0.7090,    0.8339},
+{    0.6702,    0.6823,    0.8268,    0.6250,    0.7456,    0.6782,    0.7331,    0.6205,    0.6786,    0.8033},
+{    0.6910,    0.7332,    0.8750,    0.6697,    0.8119,    0.6608,    0.7235,    0.7261,    0.7142,    0.8595},
+{    0.6467,    0.7493,    0.6821,    0.6133,    0.6857,    0.6692,    0.7265,    0.8252,    0.7259,    0.7506},
+          },
+        new double[][,]
+          {
+            new double[,] {
+{    0.4578,    0.4717,    0.2318,    0.4642,    0.3430,    0.4841,    0.4117,    0.3485,    0.4170,    0.2781},
+{    0.4373,    0.3856,    0.3808,    0.4468,    0.3645,    0.4738,    0.4080,    0.2761,    0.3758,    0.2952},
+{    0.3244,    0.3458,    0.2445,    0.3973,    0.4629,    0.3127,    0.2524,    0.4132,    0.3421,    0.2607},
+{    0.3214,    0.4679,    0.3195,    0.3750,    0.4932,    0.4535,    0.1851,    0.2015,    0.2204,    0.2712},
+{    0.1955,    0.4507,    0.3295,    0.1752,    0.4981,    0.4894,    0.3104,    0.1418,    0.1610,    0.2213},
+{    0.2084,    0.3607,    0.3861,    0.2621,    0.4877,    0.4512,    0.2138,    0.0915,    0.0887,    0.2199},
+{    0.4411,    0.3416,    0.3658,    0.4587,    0.2116,    0.4627,    0.4191,    0.4345,    0.4127,    0.2770},
+{    0.4421,    0.4335,    0.2864,    0.4687,    0.3793,    0.4365,    0.3913,    0.4710,    0.4362,    0.3160},
+{    0.4270,    0.3912,    0.2187,    0.4424,    0.3054,    0.4483,    0.4001,    0.3977,    0.4082,    0.2415},
+{    0.4570,    0.3757,    0.4337,    0.4743,    0.4310,    0.4427,    0.3974,    0.2885,    0.3980,    0.3744},
+            },
+            new double[,] {
+{    1.2905,    1.2380,    1.7324,    1.2677,    1.5604,    1.1785,    1.4203,    1.5505,    1.4074,    1.6661},
+{    1.3541,    1.4784,    1.4883,    1.3260,    1.5205,    1.2287,    1.4290,    1.6691,    1.4983,    1.6400},
+{    1.5926,    1.5553,    1.7148,    1.4532,    1.2723,    1.6120,    1.7037,    1.4168,    1.5620,    1.6918},
+{    1.5976,    1.2535,    1.6008,    1.5000,    1.1166,    1.3049,    1.7936,    1.7727,    1.7478,    1.6765},
+{    1.7803,    1.3141,    1.5839,    1.8060,    1.0609,    1.1453,    1.6157,    1.8464,    1.8234,    1.7466},
+{    1.7637,    1.5278,    1.4774,    1.6898,    1.1570,    1.3125,    1.7566,    1.9039,    1.9070,    1.7485},
+{    1.3433,    1.5628,    1.5180,    1.2873,    1.7594,    1.2733,    1.4024,    1.3619,    1.4179,    1.6679},
+{    1.3404,    1.3646,    1.6537,    1.2501,    1.4913,    1.3564,    1.4662,    1.2410,    1.3572,    1.6066},
+{    1.3820,    1.4665,    1.7501,    1.3395,    1.6239,    1.3216,    1.4470,    1.4522,    1.4285,    1.7190},
+{    1.2934,    1.4986,    1.3642,    1.2267,    1.3714,    1.3384,    1.4531,    1.6503,    1.4517,    1.5012},
+          },
+                      new double[,] {
+{   -0.0538,   -0.0611,   -0.0085,   -0.0569,   -0.0222,   -0.0696,   -0.0372,   -0.0231,   -0.0388,   -0.0131},
+{   -0.0454,   -0.0306,   -0.0295,   -0.0490,   -0.0261,   -0.0624,   -0.0362,   -0.0129,   -0.0284,   -0.0151},
+{   -0.0192,   -0.0226,   -0.0096,   -0.0334,   -0.0563,   -0.0175,   -0.0104,   -0.0377,   -0.0220,   -0.0112},
+{   -0.0187,   -0.0589,   -0.0184,   -0.0283,   -0.0788,   -0.0519,   -0.0051,   -0.0062,   -0.0076,   -0.0123},
+{   -0.0058,   -0.0506,   -0.0200,   -0.0045,   -0.0873,   -0.0745,   -0.0171,   -0.0029,   -0.0038,   -0.0077},
+{   -0.0067,   -0.0254,   -0.0307,   -0.0114,   -0.0728,   -0.0508,   -0.0071,   -0.0011,   -0.0011,   -0.0076},
+{   -0.0468,   -0.0219,   -0.0264,   -0.0542,   -0.0069,   -0.0561,   -0.0394,   -0.0444,   -0.0375,   -0.0130},
+{   -0.0472,   -0.0441,   -0.0140,   -0.0594,   -0.0292,   -0.0451,   -0.0319,   -0.0606,   -0.0450,   -0.0179},
+{   -0.0419,   -0.0319,   -0.0075,   -0.0473,   -0.0165,   -0.0496,   -0.0341,   -0.0335,   -0.0363,   -0.0094},
+{   -0.0534,   -0.0284,   -0.0441,   -0.0627,   -0.0432,   -0.0474,   -0.0334,   -0.0143,   -0.0336,   -0.0281},
+          }
+          }
+      );
+      TestCovarianceFunction(new CovarianceRQiso(), 1,
+         new double[,]
+           {
+{    6.8660,    6.8070,    7.2367,    6.8409,    7.1145,    6.7347,    6.9959,    7.1068,    6.9839,    7.1923},
+{    6.9324,    7.0474,    7.0559,    6.9038,    7.0827,    6.7961,    7.0038,    7.1944,    7.0644,    7.1739},
+{    7.1392,    7.1106,    7.2253,    7.0255,    6.8461,    7.1536,    7.2179,    6.9926,    7.1158,    7.2099},
+{    7.1429,    6.8249,    7.1453,    7.0658,    6.6525,    6.8816,    7.2752,    7.2623,    7.2467,    7.1995},
+{    7.2671,    6.8913,    7.1326,    7.2827,    6.5716,    6.6915,    7.1564,    7.3065,    7.2931,    7.2459},
+{    7.2567,    7.0886,    7.0466,    7.2086,    6.7069,    6.8897,    7.2522,    7.3388,    7.3405,    7.2471},
+{    6.9215,    7.1164,    7.0807,    6.8626,    7.2540,    6.8471,    6.9792,    6.9401,    6.9937,    7.1935},
+{    6.9185,    6.9428,    7.1836,    6.8210,    7.0584,    6.9346,    7.0369,    6.8105,    6.9354,    7.1496},
+{    6.9598,    7.0372,    7.2481,    6.9176,    7.1623,    6.8992,    7.0200,    7.0247,    7.0033,    7.2280},
+{    6.8692,    7.0646,    6.9424,    6.7937,    6.9495,    6.9165,    7.0254,    7.1813,    7.0242,    7.0668},
+           },
+         new double[][,]
+          {
+            new double[,] {
+{    0.9946,    1.1004,    0.3003,    1.0398,    0.5350,    1.2280,    0.7575,    0.5497,    0.7796,    0.3863},
+{    0.8743,    0.6614,    0.6455,    0.9263,    0.5952,    1.1197,    0.7428,    0.3823,    0.6297,    0.4217},
+{    0.4881,    0.5425,    0.3225,    0.7023,    1.0304,    0.4606,    0.3369,    0.7635,    0.5326,    0.3523},
+{    0.4810,    1.0684,    0.4764,    0.6270,    1.3705,    0.9666,    0.2253,    0.2504,    0.2810,    0.3725},
+{    0.2412,    0.9489,    0.5007,    0.2106,    1.5081,    1.3031,    0.4553,    0.1639,    0.1902,    0.2825},
+{    0.2614,    0.5840,    0.6630,    0.3549,    1.2764,    0.9519,    0.2702,    0.1000,    0.0967,    0.2802},
+{    0.8941,    0.5314,    0.5990,    1.0007,    0.2666,    1.0286,    0.7883,    0.8600,    0.7615,    0.3839},
+{    0.8995,    0.8552,    0.4030,    1.0754,    0.6409,    0.8701,    0.6811,    1.0941,    0.8687,    0.4682},
+{    0.8240,    0.6807,    0.2782,    0.9012,    0.4439,    0.9346,    0.7126,    0.7040,    0.7436,    0.3172},
+{    0.9889,    0.6293,    0.8560,    1.1240,    0.8429,    0.9031,    0.7025,    0.4076,    0.7048,    0.6252},
+            },
+            new double[,] {
+{   13.7321,   13.6139,   14.4735,   13.6818,   14.2291,   13.4693,   13.9917,   14.2135,   13.9678,   14.3846},
+{   13.8647,   14.0949,   14.1118,   13.8076,   14.1654,   13.5922,   14.0076,   14.3888,   14.1287,   14.3478},
+{   14.2784,   14.2211,   14.4506,   14.0511,   13.6922,   14.3072,   14.4358,   13.9852,   14.2316,   14.4199},
+{   14.2859,   13.6498,   14.2906,   14.1315,   13.3049,   13.7631,   14.5504,   14.5247,   14.4933,   14.3990},
+{   14.5341,   13.7827,   14.2652,   14.5654,   13.1433,   13.3830,   14.3128,   14.6130,   14.5862,   14.4918},
+{   14.5134,   14.1773,   14.0932,   14.4172,   13.4138,   13.7793,   14.5045,   14.6776,   14.6810,   14.4942},
+{   13.8429,   14.2328,   14.1613,   13.7253,   14.5081,   13.6943,   13.9583,   13.8803,   13.9873,   14.3871},
+{   13.8370,   13.8855,   14.3672,   13.6420,   14.1168,   13.8693,   14.0738,   13.6210,   13.8709,   14.2993},
+{   13.9195,   14.0743,   14.4962,   13.8352,   14.3246,   13.7985,   14.0400,   14.0493,   14.0067,   14.4561},
+{   13.7384,   14.1291,   13.8847,   13.5873,   13.8990,   13.8331,   14.0508,   14.3625,   14.0484,   14.1335},
+            },
+            new double[,] {
+{   -0.0067,   -0.0083,   -0.0006,   -0.0074,   -0.0019,   -0.0105,   -0.0038,   -0.0020,   -0.0041,   -0.0010},
+{   -0.0051,   -0.0029,   -0.0027,   -0.0058,   -0.0023,   -0.0087,   -0.0037,   -0.0009,   -0.0026,   -0.0011},
+{   -0.0015,   -0.0019,   -0.0007,   -0.0033,   -0.0073,   -0.0014,   -0.0007,   -0.0039,   -0.0019,   -0.0008},
+{   -0.0015,   -0.0078,   -0.0015,   -0.0026,   -0.0133,   -0.0064,   -0.0003,   -0.0004,   -0.0005,   -0.0009},
+{   -0.0004,   -0.0061,   -0.0016,   -0.0003,   -0.0164,   -0.0120,   -0.0013,   -0.0002,   -0.0002,   -0.0005},
+{   -0.0004,   -0.0022,   -0.0029,   -0.0008,   -0.0114,   -0.0062,   -0.0005,   -0.0001,   -0.0001,   -0.0005},
+{   -0.0054,   -0.0018,   -0.0024,   -0.0068,   -0.0005,   -0.0072,   -0.0042,   -0.0050,   -0.0039,   -0.0009},
+{   -0.0055,   -0.0049,   -0.0010,   -0.0080,   -0.0027,   -0.0051,   -0.0031,   -0.0082,   -0.0051,   -0.0014},
+{   -0.0046,   -0.0031,   -0.0005,   -0.0055,   -0.0013,   -0.0059,   -0.0034,   -0.0033,   -0.0037,   -0.0006},
+{   -0.0067,   -0.0026,   -0.0049,   -0.0087,   -0.0048,   -0.0055,   -0.0033,   -0.0011,   -0.0033,   -0.0026},                             
+                           }
+          }
+       );
+    }
+
 
     [TestMethod]
@@ -335 +447 @@
               },
           }
-      );
+      , 5e-3);
       TestCovarianceFunction(new CovariancePeriodic(), 1,
         new double[,]
@@ -393 +505 @@
               },
           }
-      );
+      , 5e-3);
     }
 
@@ -781 +893 @@
               },
           }
-      );
-    }
-
-
-    private void TestCovarianceFunction(ICovarianceFunction cf, double hypValue, double[,] expectedCov, double[][,] expectedGradients) {
+      , 5e-3);
+    }
+
+
+    private void TestCovarianceFunction(ICovarianceFunction cf, double hypValue, double[,] expectedCov, double[][,] expectedGradients, double delta = 1E-3) {
       var x = GetData();
       var xt = GetDataTest();
@@ -801 +913 @@
           actualCov[i, j] = cf.GetCovariance(i, j);
 
-      AssertEqual(expectedCov, actualCov);
+      AssertEqual(expectedCov, actualCov, delta);
 
       for (int i = 0; i < rows0; i++)
         for (int j = 0; j < rows1; j++) {
           for (int k = 0; k < nHyp; k++)
-            Assert.AreEqual(expectedGradients[k][i, j], cf.GetGradient(i, j, k), 5E-3);
+            Assert.AreEqual(expectedGradients[k][i, j], cf.GetGradient(i, j, k), delta);
         }
     }
@@ -836 +948 @@
         Assert.AreEqual(expected[i], actual[i], 1E-3);
     }
-    private void AssertEqual(double[,] expected, double[,] actual) {
+    private void AssertEqual(double[,] expected, double[,] actual, double delta = 5e-3) {
       Assert.AreEqual(expected.Length, actual.Length);
       for (int i = 0; i < expected.GetLength(0); i++)
         for (int j = 0; j < expected.GetLength(1); j++)
-          Assert.AreEqual(expected[i, j], actual[i, j], 5E-3);
+          Assert.AreEqual(expected[i, j], actual[i, j], delta);
     }
 

trunk/sources/HeuristicLab.Tests/HeuristicLab.Algorithms.DataAnalysis-3.4/GaussianProcessModelTest.cs

@@ -56 +56 @@
 
         var dHyp = model.GetHyperparameterGradients();
-        Assert.AreEqual(-2.0171e+003, dHyp[0], 1);
-        Assert.AreEqual(-248.7932, dHyp[1], 1E-2);
+        Assert.AreEqual(-248.7932, dHyp[0], 1E-2);
         var dHypCovExpected = new double[] { -0.5550e4, -5.5533e4, -0.2511e4, -2.7625e4, -1.3033e4, 0.0289e4, -2.7625e4 };
-        AssertEqual(dHypCovExpected, dHyp.Skip(2).ToArray(), 1);
+        AssertEqual(dHypCovExpected, dHyp.Skip(1).Take(7).ToArray(), 1);
+        Assert.AreEqual(-2.0171e+003, dHyp.Last(), 1);
+
 
         var predTrain = model.GetEstimatedValues(problemData.Dataset, new int[] { 0, 400 }).ToArray();
         Assert.AreEqual(310.5930, predTrain[0], 1e-3);
         Assert.AreEqual(347.9993, predTrain[1], 1e-3);
+
+        var predTrainVar = model.GetEstimatedVariance(problemData.Dataset, problemData.TrainingIndices).ToArray();
       }
 
       {
-        var hyp = new double[] { 0.716427415979145, 0.029973094285941, 0.455535210579926, 3.438647883940457, 1.464114485889487, 3.001788584487478, 3.815289323309630, 4.374914122810222, 3.001788584487478 };
+        var hyp = new double[] { 0.029973094285941, 0.455535210579926, 3.438647883940457, 1.464114485889487, 3.001788584487478, 3.815289323309630, 4.374914122810222, 3.001788584487478, 0.716427415979145 };
         var model = new GaussianProcessModel(problemData.Dataset, targetVariable, allowedInputVariables, rows, hyp,
                                              meanFunction,
@@ -74 +77 @@
 
         var dHyp = model.GetHyperparameterGradients();
-        Assert.AreEqual(0.8621, dHyp[0], 1e-3);
-        Assert.AreEqual(-0.0046, dHyp[1], 1e-3);
+        Assert.AreEqual(-0.0046, dHyp[0], 1e-3);
         var dHypCovExpected = new double[] { 0.2652, -0.2386, 0.1706, -0.1744, 0.0000, 0.0000, -0.1744 };
-        AssertEqual(dHypCovExpected, dHyp.Skip(2).ToArray(), 1e-3);
+        AssertEqual(dHypCovExpected, dHyp.Skip(1).Take(7).ToArray(), 1e-3);
+        Assert.AreEqual(0.8621, dHyp.Last(), 1e-3);
 
         var predTrain = model.GetEstimatedValues(problemData.Dataset, new int[] { 0, 400 }).ToArray();
@@ -83 +86 @@
         Assert.AreEqual(356.6076, predTrain[1], 1e-3);
       }
+
+      /*
+      {
+        // example from GPML book
+        var hyp = new double[] { 
+          341.0, // mean 341 ppm
+          // SE iso (long term trend)
+          Math.Log(67.0 / 45.0), // length scale 67 years
+          Math.Log(Math.Sqrt(66)), // magnitude 66ppm
+                    
+          // product of SEiso and periodic
+          Math.Log(90.0 / 45.0), // decay-time 90 years
+          Math.Log(Math.Sqrt(2.4)), // magnitude 2.4ppm
+
+          Math.Log(1.3), // smoothness
+          Math.Log(1), // period 1 year
+          Math.Log(Math.Sqrt(2.4)), // magnitude 2.4ppm
+
+          // short term variation
+          Math.Log(1.2 / 45.0), // typical length 1.2 years
+          Math.Log(Math.Sqrt(0.66)), // magnitude 0.66ppm
+          Math.Log(0.78), // shape (very small)
+
+          // SEiso (correlated noise)
+          Math.Log(1.6 / 45.0 / 12.0), // 1.6 months
+          Math.Log(Math.Sqrt(0.18)), // amplitude of correlated noise 0.18ppm
+          Math.Log(Math.Sqrt(0.19)),  // theta11 0.19ppm noise
+          };
+
+        covarianceFunction.Terms.Add(new CovarianceRQiso());
+        covarianceFunction.Terms.Add(new CovarianceSEiso()); // correlated noise
+        var model = new GaussianProcessModel(problemData.Dataset, targetVariable, allowedInputVariables, Enumerable.Range(0, 545), hyp,
+                                             new MeanConst(),
+                                             covarianceFunction);
+        Assert.AreEqual(-108.5, model.NegativeLogLikelihood, 1);
+      }
+       */
     }