Changeset 14899

Timestamp:

04/29/17 17:35:55 (8 years ago)

Author:

gkronber

Message:

#2782: implemented calculation of LOO predictive probability for Gaussian process regression

Location:

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4

Files:

• GaussianProcess/GaussianProcessModel.cs (modified) (3 diffs)
• GaussianProcess/GaussianProcessModelCreator.cs (modified) (4 diffs)
• GaussianProcess/GaussianProcessRegressionModelCreator.cs (modified) (1 diff)
• GaussianProcess/GaussianProcessRegressionSolutionCreator.cs (modified) (2 diffs)
• Interfaces/IGaussianProcessModel.cs (modified) (1 diff)
• Interfaces/IGaussianProcessModelCreator.cs (modified) (2 diffs)

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs

@@ -46 +46 @@
 
     [Storable]
+    private double negativeLooPredictiveProbability;
+    public double NegativeLooPredictiveProbability {
+      get { return negativeLooPredictiveProbability; }
+    }
+
+    [Storable]
     private double[] hyperparameterGradients;
     public double[] HyperparameterGradients {
@@ -128 +134 @@
       this.trainingDataset = cloner.Clone(original.trainingDataset);
       this.negativeLogLikelihood = original.negativeLogLikelihood;
+      this.negativeLooPredictiveProbability = original.negativeLooPredictiveProbability;
       this.sqrSigmaNoise = original.sqrSigmaNoise;
       if (original.meanParameter != null) {
@@ -217 +224 @@
       alglib.spdmatrixcholeskyinverse(ref lCopy, n, false, out info, out matInvRep);
       if (info != 1) throw new ArgumentException("Can't invert matrix to calculate gradients.");
+
+      // LOOCV log predictive probability (GPML page 116 and 117)
+      var sumLoo = 0.0;
+      var ki = new double[n];
+      for (int i = 0; i < n; i++) {
+        for (int j = 0; j < n; j++) ki[j] = cov.Covariance(x, i, j);
+        var yi = Util.ScalarProd(ki, alpha);
+        // r = [inv(K)]_i,i
+        var yi_loo = yi - alpha[i] / lCopy[i, i] / sqrSigmaNoise;
+        var s2_loo = sqrSigmaNoise / lCopy[i, i];
+        var err = ym[i] - yi_loo;
+        var nll_loo = Math.Log(s2_loo) + err * err / s2_loo;
+        sumLoo += nll_loo;
+      }
+      sumLoo += n * Math.Log(2 * Math.PI);
+      negativeLooPredictiveProbability = 0.5 * sumLoo;
+
       for (int i = 0; i < n; i++) {
         for (int j = 0; j <= i; j++)

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModelCreator.cs

@@ -37 +37 @@
     private const string ModelParameterName = "Model";
     private const string NegativeLogLikelihoodParameterName = "NegativeLogLikelihood";
+    private const string NegativeLogPredictiveProbabilityParameterName = "NegativeLogPredictiveProbability (LOOCV)";
     private const string HyperparameterGradientsParameterName = "HyperparameterGradients";
     protected const string ScaleInputValuesParameterName = "ScaleInputValues";
@@ -60 +61 @@
     public ILookupParameter<DoubleValue> NegativeLogLikelihoodParameter {
       get { return (ILookupParameter<DoubleValue>)Parameters[NegativeLogLikelihoodParameterName]; }
+    }
+    public ILookupParameter<DoubleValue> NegativeLogPredictiveProbabilityParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters[NegativeLogPredictiveProbabilityParameterName]; }
     }
     public ILookupParameter<BoolValue> ScaleInputValuesParameter {
@@ -86 +90 @@
       Parameters.Add(new LookupParameter<RealVector>(HyperparameterGradientsParameterName, "The gradients of the hyperparameters for the produced Gaussian process model (necessary for hyperparameter optimization)"));
       Parameters.Add(new LookupParameter<DoubleValue>(NegativeLogLikelihoodParameterName, "The negative log-likelihood of the produced Gaussian process model given the data."));
+      Parameters.Add(new LookupParameter<DoubleValue>(NegativeLogPredictiveProbabilityParameterName, "The leave-one-out-cross-validation negative log predictive probability of the produced Gaussian process model given the data."));
 
 
@@ -100 +105 @@
         Parameters[ScaleInputValuesParameterName].Hidden = true;
       }
+      if (!Parameters.ContainsKey(NegativeLogPredictiveProbabilityParameterName)) {
+        Parameters.Add(new LookupParameter<DoubleValue>(NegativeLogPredictiveProbabilityParameterName,
+          "The leave-one-out-cross-validation negative log predictive probability of the produced Gaussian process model given the data."));
+      }
     }
   }

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs

@@ -65 +65 @@
         ModelParameter.ActualValue = model;
         NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(model.NegativeLogLikelihood);
+        NegativeLogPredictiveProbabilityParameter.ActualValue = new DoubleValue(model.NegativeLooPredictiveProbability);
         HyperparameterGradientsParameter.ActualValue = new RealVector(model.HyperparameterGradients);
         return base.Apply();

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs

@@ -41 +41 @@
     private const string TestRSquaredResultName = "Test R²";
     private const string CreateSolutionParameterName = "CreateSolution";
+    private const string NegLogPredictiveProbability = "NegativeLogPredictiveProbability (LOO-CV)";
 
     #region Parameter Properties
@@ -108 +109 @@
                                  "The Pearson's R² of the Gaussian process solution on the test partition.",
                                  new DoubleValue(s.TestRSquared)));
+          results.Add(new Result(NegLogPredictiveProbability, 
+                                 "The leave-one-out-cross-validation negative log predictive probability.", 
+                                 new DoubleValue(m.NegativeLooPredictiveProbability)));
         } else {
           results[SolutionParameterName].Value = s;
           results[TrainingRSquaredResultName].Value = new DoubleValue(s.TrainingRSquared);
           results[TestRSquaredResultName].Value = new DoubleValue(s.TestRSquared);
+          results[NegLogPredictiveProbability].Value = new DoubleValue(m.NegativeLooPredictiveProbability);
         }
       }

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModel.cs

@@ -28 +28 @@
   public interface IGaussianProcessModel : IConfidenceRegressionModel {
     double NegativeLogLikelihood { get; }
+    double NegativeLooPredictiveProbability { get; }
     double SigmaNoise { get; }
     IMeanFunction MeanFunction { get; }

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModelCreator.cs

@@ -23 +23 @@
 using HeuristicLab.Data;
 using HeuristicLab.Encodings.RealVectorEncoding;
-using HeuristicLab.Problems.DataAnalysis;
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
@@ -36 +35 @@
     ILookupParameter<RealVector> HyperparameterGradientsParameter { get; }
     ILookupParameter<DoubleValue> NegativeLogLikelihoodParameter { get; }
+    ILookupParameter<DoubleValue> NegativeLogPredictiveProbabilityParameter { get; }
   }
 }