Changeset 17991 for branches/3128_Prediction_Intervals/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression

Timestamp:

06/16/21 21:35:37 (3 years ago)

Author:

gkronber

Message:

#3128: first dump of exploratory work-in-progress code to make sure the working copy is not lost.

Location:

branches/3128_Prediction_Intervals/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4

Files:

• Interfaces/ISymbolicRegressionModel.cs (modified) (1 diff)
• Interfaces/ISymbolicRegressionSolution.cs (modified) (1 diff)
• SingleObjective/SymbolicRegressionSingleObjectiveTrainingBestSolutionAnalyzer.cs (modified) (2 diffs)
• SymbolicRegressionModel.cs (modified) (4 diffs)
• SymbolicRegressionSolution.cs (modified) (3 diffs)

branches/3128_Prediction_Intervals/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/Interfaces/ISymbolicRegressionModel.cs

@@ -24 +24 @@
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Regression {
   [StorableType("a411e2b5-f926-41a6-b55b-1aef862db2fb")]
-  public interface ISymbolicRegressionModel : IRegressionModel, ISymbolicDataAnalysisModel {
+  public interface ISymbolicRegressionModel : IRegressionModel, ISymbolicDataAnalysisModel, IConfidenceRegressionModel {
     void Scale(IRegressionProblemData problemData);
   }

branches/3128_Prediction_Intervals/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/Interfaces/ISymbolicRegressionSolution.cs

@@ -25 +25 @@
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Regression {
   [StorableType("dff1a450-e958-454f-bf8e-6b763fdcaff3")]
-  public interface ISymbolicRegressionSolution : IRegressionSolution, ISymbolicDataAnalysisSolution {
+  public interface ISymbolicRegressionSolution : IRegressionSolution, ISymbolicDataAnalysisSolution, IConfidenceRegressionSolution {
     new ISymbolicRegressionModel Model { get; }
   }

branches/3128_Prediction_Intervals/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SingleObjective/SymbolicRegressionSingleObjectiveTrainingBestSolutionAnalyzer.cs

@@ -25 +25 @@
 using HeuristicLab.Parameters;
 using HEAL.Attic;
+using HeuristicLab.Data;
+using System.Collections.Generic;
+using System;
+using System.Linq;
+using HeuristicLab.Analysis.Statistics;
 
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Regression {
@@ -63 +68 @@
 
     protected override ISymbolicRegressionSolution CreateSolution(ISymbolicExpressionTree bestTree, double bestQuality) {
-      var model = new SymbolicRegressionModel(ProblemDataParameter.ActualValue.TargetVariable, (ISymbolicExpressionTree)bestTree.Clone(), SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, EstimationLimitsParameter.ActualValue.Lower, EstimationLimitsParameter.ActualValue.Upper);
+
+      // HACK: create model first for scaling, then calculate statistics and create a new model with prediction intervals
+      var model = new SymbolicRegressionModel(ProblemDataParameter.ActualValue.TargetVariable,
+        (ISymbolicExpressionTree)bestTree.Clone(),
+        SymbolicDataAnalysisTreeInterpreterParameter.ActualValue,
+        EstimationLimitsParameter.ActualValue.Lower,
+        EstimationLimitsParameter.ActualValue.Upper);
       if (ApplyLinearScalingParameter.ActualValue.Value) model.Scale(ProblemDataParameter.ActualValue);
-      return new SymbolicRegressionSolution(model, (IRegressionProblemData)ProblemDataParameter.ActualValue.Clone());
+
+      // use scaled tree
+      CalculateParameterCovariance(model.SymbolicExpressionTree, ProblemDataParameter.ActualValue, SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, out var cov, out var sigma);
+      var predIntervalModel = new SymbolicRegressionModel(ProblemDataParameter.ActualValue.TargetVariable,
+        (ISymbolicExpressionTree)model.SymbolicExpressionTree.Clone(),
+        SymbolicDataAnalysisTreeInterpreterParameter.ActualValue,
+        EstimationLimitsParameter.ActualValue.Lower,
+        EstimationLimitsParameter.ActualValue.Upper, parameterCovariance: cov, sigma: sigma);
+
+      return new SymbolicRegressionSolution(predIntervalModel, (IRegressionProblemData)ProblemDataParameter.ActualValue.Clone());
+    }
+
+    private void CalculateParameterCovariance(ISymbolicExpressionTree tree, IRegressionProblemData problemData, ISymbolicDataAnalysisExpressionTreeInterpreter interpreter, out double[,] cov, out double sigma) {
+      var y_pred = interpreter.GetSymbolicExpressionTreeValues(tree, problemData.Dataset, problemData.TrainingIndices).ToArray();
+      var residuals = problemData.TargetVariableTrainingValues.Zip(y_pred, (yi, y_pred_i) => yi - y_pred_i).ToArray();
+
+      var paramNodes = new List<ISymbolicExpressionTreeNode>();
+      var coeffList = new List<double>();
+      foreach (var node in tree.IterateNodesPostfix()) {
+        if (node is ConstantTreeNode constNode) {
+          paramNodes.Add(constNode);
+          coeffList.Add(constNode.Value);
+        } else if (node is VariableTreeNode varNode) {
+          paramNodes.Add(varNode);
+          coeffList.Add(varNode.Weight);
+        }
+      }
+      var coeff = coeffList.ToArray();
+      var numParams = coeff.Length;
+
+      var rows = problemData.TrainingIndices.ToArray();
+      var dcoeff = new double[rows.Length, numParams];
+      TreeToAutoDiffTermConverter.TryConvertToAutoDiff(tree, makeVariableWeightsVariable: true, addLinearScalingTerms: false,
+        out var parameters, out var initialConstants, out var func, out var func_grad);
+      if (initialConstants.Zip(coeff, (ici, coi) => ici != coi).Any(t => t)) throw new InvalidProgramException();
+      var ds = problemData.Dataset;
+      var x_r = new double[parameters.Count];
+      for (int r = 0; r < rows.Length; r++) {
+        // copy row
+        for (int c = 0; c < parameters.Count; c++) {
+          x_r[c] = ds.GetDoubleValue(parameters[c].variableName, rows[r]);
+        }
+        var tup = func_grad(coeff, x_r);
+        for (int c = 0; c < numParams; c++) {
+          dcoeff[r, c] = tup.Item1[c];
+        }
+      }
+
+      var stats = Statistics.CalculateLinearModelStatistics(dcoeff, coeff, residuals);
+      cov = stats.CovMx;
+      sigma = stats.sigma;
     }
   }

branches/3128_Prediction_Intervals/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SymbolicRegressionModel.cs

@@ -26 +26 @@
 using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
 using HEAL.Attic;
+using HeuristicLab.Data;
+using System.Linq;
 
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Regression {
@@ -45 +47 @@
     }
 
+    [Storable]
+    private readonly double[,] parameterCovariance;
+    [Storable]
+    private readonly double sigma;
+
     [StorableConstructor]
     protected SymbolicRegressionModel(StorableConstructorFlag _) : base(_) {
@@ -53 +60 @@
       : base(original, cloner) {
       this.targetVariable = original.targetVariable;
+      this.parameterCovariance = original.parameterCovariance; // immutable
+      this.sigma = original.sigma;
     }
 
     public SymbolicRegressionModel(string targetVariable, ISymbolicExpressionTree tree,
       ISymbolicDataAnalysisExpressionTreeInterpreter interpreter,
-      double lowerEstimationLimit = double.MinValue, double upperEstimationLimit = double.MaxValue)
+      double lowerEstimationLimit = double.MinValue, double upperEstimationLimit = double.MaxValue, double[,] parameterCovariance = null, double sigma = 0.0)
       : base(tree, interpreter, lowerEstimationLimit, upperEstimationLimit) {
       this.targetVariable = targetVariable;
+      if (parameterCovariance != null)
+        this.parameterCovariance = (double[,])parameterCovariance.Clone();
+      this.sigma = sigma;
     }
 
@@ -69 +81 @@
       return Interpreter.GetSymbolicExpressionTreeValues(SymbolicExpressionTree, dataset, rows)
         .LimitToRange(LowerEstimationLimit, UpperEstimationLimit);
+    }
+
+    public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
+      // must work with a copy because we change tree nodes
+      var treeCopy = (ISymbolicExpressionTree)SymbolicExpressionTree.Clone();
+      // uses sampling to produce prediction intervals
+      alglib.hqrndseed(31415, 926535, out var state);
+      var cov = parameterCovariance;
+      if (cov == null || cov.Length == 0) return rows.Select(_ => 0.0);
+      var n = 30;
+      var M = rows.Select(_ => new double[n]).ToArray();
+      var paramNodes = new List<ISymbolicExpressionTreeNode>();
+      var coeffList = new List<double>();
+      // HACK: skip linear scaling parameters because the analyzer doesn't use them (and they are likely correlated with the remaining parameters)
+      // only works with linear scaling
+      if (!(treeCopy.Root.GetSubtree(0).GetSubtree(0).Symbol is Addition) ||
+          !(treeCopy.Root.GetSubtree(0).GetSubtree(0).GetSubtree(0).Symbol is Multiplication)) 
+        throw new NotImplementedException("prediction intervals are implemented only for linear scaling");
+
+      foreach (var node in treeCopy.Root.GetSubtree(0).GetSubtree(0).IterateNodesPostfix()) {
+        if (node is ConstantTreeNode constNode) {
+          paramNodes.Add(constNode);
+          coeffList.Add(constNode.Value);
+        } else if (node is VariableTreeNode varNode) {
+          paramNodes.Add(varNode);
+          coeffList.Add(varNode.Weight);
+        }
+      }
+      var coeff = coeffList.ToArray();
+      var numParams = coeff.Length;
+      if (cov.GetLength(0) != numParams) throw new InvalidProgramException();
+
+      // TODO: probably we do not need to sample but can instead use a first-order or second-order approximation of f
+      // see http://sia.webpopix.org/nonlinearRegression.html
+      // also see https://rmazing.wordpress.com/2013/08/26/predictnls-part-2-taylor-approximation-confidence-intervals-for-nls-models/
+      // https://www.rdocumentation.org/packages/propagate/versions/1.0-4/topics/predictNLS
+      double[] p = new double[numParams];
+      for (int i = 0; i < 30; i++) {
+        // sample and update parameter vector delta is 
+        alglib.hqrndnormalv(state, numParams, out var delta);
+        alglib.rmatrixmv(numParams, numParams, cov, 0, 0, 0, delta, 0, ref p, 0);
+        for (int j = 0; j < numParams; j++) {
+          if (paramNodes[j] is ConstantTreeNode constNode) constNode.Value = coeff[j] + p[j];
+          else if (paramNodes[j] is VariableTreeNode varNode) varNode.Weight = coeff[j] + p[j];
+        }
+        var r = 0;
+        var estimatedValues = Interpreter.GetSymbolicExpressionTreeValues(treeCopy, dataset, rows).LimitToRange(LowerEstimationLimit, UpperEstimationLimit);
+
+        foreach (var pred in estimatedValues) {
+          M[r++][i] = pred;
+        }
+      }
+
+      // reset parameters
+      for (int j = 0; j < numParams; j++) {
+        if (paramNodes[j] is ConstantTreeNode constNode) constNode.Value = coeff[j];
+        else if (paramNodes[j] is VariableTreeNode varNode) varNode.Weight = coeff[j];
+      }
+      var sigma2 = sigma * sigma;
+      return M.Select(M_i => M_i.Variance() + sigma2).ToArray();
     }
 

branches/3128_Prediction_Intervals/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SymbolicRegressionSolution.cs

@@ -20 +20 @@
 #endregion
 
+using System.Collections.Generic;
 using System.Linq;
 using HEAL.Attic;
@@ -112 +113 @@
     }
 
-
+    IConfidenceRegressionModel IConfidenceRegressionSolution.Model => Model;
 
     [StorableConstructor]
@@ -215 +216 @@
       return intervalEvaluation;
     }
+
+    public IEnumerable<double> EstimatedVariances => GetEstimatedVariances(ProblemData.AllIndices);
+
+    public IEnumerable<double> EstimatedTrainingVariances => GetEstimatedVariances(ProblemData.TestIndices);
+
+    public IEnumerable<double> EstimatedTestVariances => GetEstimatedVariances(ProblemData.TestIndices);
+
+
+    public IEnumerable<double> GetEstimatedVariances(IEnumerable<int> rows) {
+      return Model.GetEstimatedVariances(ProblemData.Dataset, rows);
+    }
   }
 }