Changeset 15188

Timestamp:

07/10/17 19:19:26 (7 years ago)

Author:

gkronber

Message:

#2782: merged r14899,r14918,r15160,r15163,r15165,r15187 from trunk to stable

Location:

stable

Files:

• . (modified) (1 prop)
• HeuristicLab.Algorithms.DataAnalysis (modified) (1 prop)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs (modified) (3 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModelCreator.cs (modified) (4 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs (modified) (2 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModel.cs (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModelCreator.cs (modified) (2 diffs)

stable/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs

@@ -46 +46 @@
 
     [Storable]
+    private double loocvNegLogPseudoLikelihood;
+    public double LooCvNegativeLogPseudoLikelihood {
+      get { return loocvNegLogPseudoLikelihood; }
+    }
+
+    [Storable]
     private double[] hyperparameterGradients;
     public double[] HyperparameterGradients {
@@ -128 +134 @@
       this.trainingDataset = cloner.Clone(original.trainingDataset);
       this.negativeLogLikelihood = original.negativeLogLikelihood;
+      this.loocvNegLogPseudoLikelihood = original.loocvNegLogPseudoLikelihood;
       this.sqrSigmaNoise = original.sqrSigmaNoise;
       if (original.meanParameter != null) {
@@ -217 +224 @@
       alglib.spdmatrixcholeskyinverse(ref lCopy, n, false, out info, out matInvRep);
       if (info != 1) throw new ArgumentException("Can't invert matrix to calculate gradients.");
+
+      // LOOCV log pseudo-likelihood (or log predictive probability) (GPML page 116 and 117)
+      var sumLoo = 0.0;
+      var ki = new double[n];
+      for (int i = 0; i < n; i++) {
+        for (int j = 0; j < n; j++) ki[j] = cov.Covariance(x, i, j);
+        ki[i] += sqrSigmaNoise;
+
+        var yi = Util.ScalarProd(ki, alpha);
+        var yi_loo = yi - alpha[i] / (lCopy[i, i] / sqrSigmaNoise);
+        var s2_loo = 1.0 / (lCopy[i, i] / sqrSigmaNoise);
+        var err = ym[i] - yi_loo;
+        var nll_loo = 0.5 * Math.Log(2 * Math.PI * s2_loo) + 0.5 * err * err / s2_loo;
+        sumLoo += nll_loo;
+      }
+      loocvNegLogPseudoLikelihood = sumLoo;
+
       for (int i = 0; i < n; i++) {
         for (int j = 0; j <= i; j++)

stable/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModelCreator.cs

@@ -37 +37 @@
     private const string ModelParameterName = "Model";
     private const string NegativeLogLikelihoodParameterName = "NegativeLogLikelihood";
+    private const string NegativeLogPseudoLikelihoodParameterName = "NegativeLogPseudoLikelihood (LOOCV)";
     private const string HyperparameterGradientsParameterName = "HyperparameterGradients";
     protected const string ScaleInputValuesParameterName = "ScaleInputValues";
@@ -60 +61 @@
     public ILookupParameter<DoubleValue> NegativeLogLikelihoodParameter {
       get { return (ILookupParameter<DoubleValue>)Parameters[NegativeLogLikelihoodParameterName]; }
+    }
+    public ILookupParameter<DoubleValue> NegativeLogPseudoLikelihoodParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters[NegativeLogPseudoLikelihoodParameterName]; }
     }
     public ILookupParameter<BoolValue> ScaleInputValuesParameter {
@@ -86 +90 @@
       Parameters.Add(new LookupParameter<RealVector>(HyperparameterGradientsParameterName, "The gradients of the hyperparameters for the produced Gaussian process model (necessary for hyperparameter optimization)"));
       Parameters.Add(new LookupParameter<DoubleValue>(NegativeLogLikelihoodParameterName, "The negative log-likelihood of the produced Gaussian process model given the data."));
+      Parameters.Add(new LookupParameter<DoubleValue>(NegativeLogPseudoLikelihoodParameterName, "The leave-one-out-cross-validation negative log pseudo-likelihood of the produced Gaussian process model given the data."));
 
 
@@ -100 +105 @@
         Parameters[ScaleInputValuesParameterName].Hidden = true;
       }
+      if (!Parameters.ContainsKey(NegativeLogPseudoLikelihoodParameterName)) {
+        Parameters.Add(new LookupParameter<DoubleValue>(NegativeLogPseudoLikelihoodParameterName,
+          "The leave-one-out-cross-validation negative log pseudo-likelihood of the produced Gaussian process model given the data."));
+      }
     }
   }

stable/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs

@@ -65 +65 @@
         ModelParameter.ActualValue = model;
         NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(model.NegativeLogLikelihood);
+        NegativeLogPseudoLikelihoodParameter.ActualValue = new DoubleValue(model.LooCvNegativeLogPseudoLikelihood);
         HyperparameterGradientsParameter.ActualValue = new RealVector(model.HyperparameterGradients);
         return base.Apply();

stable/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs

@@ -41 +41 @@
     private const string TestRSquaredResultName = "Test R²";
     private const string CreateSolutionParameterName = "CreateSolution";
+    private const string NegLogPseudoLikelihood = "Negative log pseudo-likelihood (LOO-CV)";
 
     #region Parameter Properties
@@ -108 +109 @@
                                  "The Pearson's R² of the Gaussian process solution on the test partition.",
                                  new DoubleValue(s.TestRSquared)));
+          results.Add(new Result(NegLogPseudoLikelihood,
+                                 "The negative log pseudo-likelihood (from leave-one-out-cross-validation).", 
+                                 new DoubleValue(m.LooCvNegativeLogPseudoLikelihood)));
         } else {
           results[SolutionParameterName].Value = s;
           results[TrainingRSquaredResultName].Value = new DoubleValue(s.TrainingRSquared);
           results[TestRSquaredResultName].Value = new DoubleValue(s.TestRSquared);
+          results[NegLogPseudoLikelihood].Value = new DoubleValue(m.LooCvNegativeLogPseudoLikelihood);
         }
       }

stable/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModel.cs

@@ -28 +28 @@
   public interface IGaussianProcessModel : IConfidenceRegressionModel {
     double NegativeLogLikelihood { get; }
+    double LooCvNegativeLogPseudoLikelihood { get; }
     double SigmaNoise { get; }
     IMeanFunction MeanFunction { get; }

stable/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModelCreator.cs

@@ -23 +23 @@
 using HeuristicLab.Data;
 using HeuristicLab.Encodings.RealVectorEncoding;
-using HeuristicLab.Problems.DataAnalysis;
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
@@ -36 +35 @@
     ILookupParameter<RealVector> HyperparameterGradientsParameter { get; }
     ILookupParameter<DoubleValue> NegativeLogLikelihoodParameter { get; }
+    ILookupParameter<DoubleValue> NegativeLogPseudoLikelihoodParameter { get; }
   }
 }