Changeset 17933

Timestamp:

04/09/21 19:51:38 (3 years ago)

Author:

gkronber

Message:

#3116: added GAM base-learners for 2d cubic splines and 3d functions (using alglib RBF model)

Location:

branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4

Files:

• GAM/AlglibRbfModel.cs (added)
• GAM/GeneralizedAdditiveModelAlgorithm.cs (modified) (12 diffs)
• GAM/Spline1dModel.cs (modified) (5 diffs)
• GAM/Spline2dModel.cs (added)
• HeuristicLab.Algorithms.DataAnalysis-3.4.csproj (modified) (1 diff)

branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/GAM/GeneralizedAdditiveModelAlgorithm.cs

@@ -47 +47 @@
     private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
     private const string CreateSolutionParameterName = "CreateSolution";
+    private const string BaseLearnerParameterName = "BaseLearner";
+    private const string Spline1dBaseLearner = "1d cubic regression spline (penalized)";
+    private const string Spline2dBaseLearner = "2d cubic spline";
+    private const string Spline3dBaseLearner = "3d cubic spline";
 
     #endregion
@@ -72 +76 @@
     }
 
+    public IConstrainedValueParameter<StringValue> BaseLearnerParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[BaseLearnerParameterName]; }
+    }
     #endregion
 
@@ -101 +108 @@
     }
 
+    public string BaseLearner {
+      get { return BaseLearnerParameter.Value.Value; }
+      set { BaseLearnerParameter.Value = BaseLearnerParameter.ValidValues.Single(vv => vv.Value == value); }
+    }
     #endregion
+
+    public override bool SupportsStop => true;
 
     [StorableConstructor]
@@ -122 +135 @@
         "Number of iterations. Try a large value and check convergence of the error over iterations. Usually, only a few iterations (e.g. 10) are needed for convergence.", new IntValue(10)));
       Parameters.Add(new FixedValueParameter<DoubleValue>(LambdaParameterName,
-        "The penalty parameter for the penalized regression splines. Set to a value between -8 (weak smoothing) and 8 (strong smooting). Usually, a value between -4 and 4 should be fine", new DoubleValue(3)));
+        "The penalty parameter for the penalized regression splines. Set to a value between -8 (weak smoothing) and 8 (strong smoothing). Usually, a value between -4 and 4 should be fine", new DoubleValue(3)));
       Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName,
         "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
@@ -130 +143 @@
         "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
       Parameters[CreateSolutionParameterName].Hidden = true;
+      var validBaseLearners = new ItemSet<StringValue>(new StringValue[] { new StringValue(Spline1dBaseLearner).AsReadOnly(), new StringValue(Spline2dBaseLearner).AsReadOnly(), new StringValue(Spline3dBaseLearner).AsReadOnly()});
+      Parameters.Add(new ConstrainedValueParameter<StringValue>("BaseLearner", "The model to use for the additive functions", validBaseLearners, validBaseLearners.First()));
     }
 
@@ -148 +163 @@
       var inputVars = problemData.AllowedInputVariables.ToArray();
 
-      int nTerms = inputVars.Length;
+      var maxInteractions = 0;
+      if (BaseLearner == Spline1dBaseLearner) maxInteractions = 1;
+      else if (BaseLearner == Spline2dBaseLearner) maxInteractions = 2;
+      else if (BaseLearner == Spline3dBaseLearner) maxInteractions = 3;
+      else throw new ArgumentException("Unknown base learner.");
+      var varTuples = EnumerableExtensions.Combinations(inputVars, maxInteractions).ToArray();
+      var varTupleNames = varTuples.Select(comb => string.Join(" ", comb)).ToArray();
+      int nTerms = varTuples.Length;
 
       #region init results
@@ -167 +189 @@
 
       // calculate table with residual contributions of each term
-      var rssTable = new DoubleMatrix(nTerms, 1, new string[] { "RSS" }, inputVars);
+      var rssTable = new DoubleMatrix(nTerms, 1, new string[] { "RSS" }, varTupleNames);
       Results.Add(new Result("RSS Values", rssTable));
       #endregion
@@ -194 +216 @@
         idx.ShuffleInPlace(rand);
         foreach (var inputIdx in idx) {
-          var inputVar = inputVars[inputIdx];
+          if (cancellationToken.IsCancellationRequested) break;
+          var fInputVars = varTuples[inputIdx];
           // first remove the effect of the previous model for the inputIdx (by adding the output of the current model to the residual)
           AddInPlace(res, f[inputIdx].GetEstimatedValues(ds, trainRows));
@@ -200 +223 @@
 
           rssTable[inputIdx, 0] = MSE(res);
-          f[inputIdx] = RegressSpline(problemData, inputVar, res, lambda);
+          if (fInputVars.Count() == 1) {
+            f[inputIdx] = Regress1dSpline(problemData, fInputVars.Single(), res, lambda);
+          } else if (fInputVars.Count() == 2) {
+            f[inputIdx] = Regress2dSpline(problemData, fInputVars, res, lambda);
+          } else if (fInputVars.Count() == 3) {
+            f[inputIdx] = Regress3dSpline(problemData, fInputVars, res, lambda);
+          } else {
+            throw new ArgumentException();
+          }
 
           SubtractInPlace(res, f[inputIdx].GetEstimatedValues(ds, trainRows));
@@ -217 +248 @@
         var model = new RegressionEnsembleModel(f.Concat(new[] { new ConstantModel(avgY, problemData.TargetVariable) }));
         model.AverageModelEstimates = false;
-        var solution = model.CreateRegressionSolution((IRegressionProblemData)problemData.Clone());        
+        var solution = model.CreateRegressionSolution((IRegressionProblemData)problemData.Clone());
         Results.Add(new Result("Ensemble solution", solution));
       }
@@ -223 +254 @@
 
     public static double MSE(IEnumerable<double> residuals) {
-      var mse  = residuals.Select(r => r * r).Average();
+      var mse = residuals.Select(r => r * r).Average();
       return mse;
     }
@@ -233 +264 @@
     }
 
-    private IRegressionModel RegressSpline(IRegressionProblemData problemData, string inputVar, double[] target, double lambda) {
+    private IRegressionModel Regress1dSpline(IRegressionProblemData problemData, string inputVar, double[] target, double lambda) {
       var x = problemData.Dataset.GetDoubleValues(inputVar, problemData.TrainingIndices).ToArray();
       var y = (double[])target.Clone();
-      int info;
       alglib.spline1dinterpolant s;
-      alglib.spline1dfitreport rep;
       int numKnots = (int)Math.Min(50, 3 * Math.Sqrt(x.Length)); // heuristic for number of knots  (Elements of Statistical Learning)
 
-      alglib.spline1dfitpenalized(x, y, numKnots, lambda, out info, out s, out rep);
+      alglib.spline1dfitpenalized(x, y, numKnots, lambda, out _, out s, out _);
 
       return new Spline1dModel(s.innerobj, problemData.TargetVariable, inputVar);
+    }
+
+    private IRegressionModel Regress2dSpline(IRegressionProblemData problemData, IEnumerable<string> inputVars, double[] target, double lambda) {
+      var x = problemData.Dataset.ToArray(inputVars, problemData.TrainingIndices);
+      var xy = x.HorzCat(target);
+
+      alglib.spline2dbuilder builder;
+      int d = 1; // scalar output
+      int numKnots = (int)Math.Min(50, 3 * Math.Sqrt(x.Length)); // heuristic for number of knots  (Elements of Statistical Learning)
+
+      alglib.spline2dbuildercreate(d, out builder);
+      alglib.spline2dbuildersetpoints(builder, xy, xy.GetLength(0));
+      alglib.spline2dbuildersetgrid(builder, numKnots, numKnots);
+      alglib.spline2dbuildersetalgoblocklls(builder, Math.Exp(lambda));
+
+      //
+      // Now we are ready to fit and evaluate our results
+      //
+      alglib.spline2dinterpolant s;
+      alglib.spline2dfit(builder, out s, out _);
+      return new Spline2dModel(s, problemData.TargetVariable, inputVars.ElementAt(0), inputVars.ElementAt(1));
+    }
+
+    private IRegressionModel Regress3dSpline(IRegressionProblemData problemData, IEnumerable<string> inputVars, double[] target, double lambda) {
+      var x = problemData.Dataset.ToArray(inputVars, problemData.TrainingIndices);
+      var xy = x.HorzCat(target);
+
+      var rbase = 100.0;
+      var nlayers = 3;
+
+      alglib.rbfmodel model;
+      alglib.rbfcreate(nx: 3, ny: 1, out model);
+      alglib.rbfsetpoints(model, xy);
+      alglib.rbfsetalgohierarchical(model, rbase, nlayers, Math.Exp(lambda));
+      alglib.rbfbuildmodel(model, out _);
+
+      return new AlglibRbfModel(model, problemData.TargetVariable, inputVars.ElementAt(0), inputVars.ElementAt(1), inputVars.ElementAt(2));
     }
 

branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/GAM/Spline1dModel.cs

@@ -29 +29 @@
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
+
   [Item("Spline model (1d)",
     "Univariate spline model (wrapper for alglib.spline1dmodel)")]
@@ -40 +41 @@
       set {
         if (value.Length > 1) throw new ArgumentException("A one-dimensional spline model supports only one input variable.");
-        inputVariable = value[0];
+        x1 = value[0];
       }
     }
 
     [Storable]
-    private string inputVariable;
-    public override IEnumerable<string> VariablesUsedForPrediction => new[] { inputVariable };
+    private string x1;
+    public override IEnumerable<string> VariablesUsedForPrediction => new[] { x1 };
 
     [StorableConstructor]
@@ -54 +55 @@
 
     private Spline1dModel(Spline1dModel orig, Cloner cloner) : base(orig, cloner) {
-      this.inputVariable = orig.inputVariable;
+      this.x1 = orig.x1;
       this.interpolant = (alglib.spline1d.spline1dinterpolant)orig.interpolant.make_copy();
     }
@@ -60 +61 @@
       : base(targetVar, $"Spline model ({inputVar})") {
       this.interpolant = (alglib.spline1d.spline1dinterpolant)interpolant.make_copy();
-      this.inputVariable = inputVar;      
+      this.x1 = inputVar;
     }
 
@@ -67 +68 @@
 
     public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
-      var solution =  new RegressionSolution(this, (IRegressionProblemData)problemData.Clone());
-      solution.Name = $"Regression Spline ({inputVariable})";
+      var solution = new RegressionSolution(this, (IRegressionProblemData)problemData.Clone());
+      solution.Name = $"Regression Spline ({x1})";
 
       return solution;
     }
 
-    public double GetEstimatedValue(double x) => alglib.spline1d.spline1dcalc(interpolant, x);
+    public double GetEstimatedValue(double x) => alglib.spline1d.spline1dcalc(interpolant, x, null);
 
     public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
-      return dataset.GetDoubleValues(inputVariable, rows).Select(GetEstimatedValue);
+      return dataset.GetDoubleValues(x1, rows).Select(GetEstimatedValue);
     }
 

branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj

@@ -142 +142 @@
     <Compile Include="FixedDataAnalysisAlgorithm.cs" />
     <Compile Include="GAM\GeneralizedAdditiveModelAlgorithm.cs" />
+    <Compile Include="GAM\AlglibRbfModel.cs" />
+    <Compile Include="GAM\Spline2dModel.cs" />
     <Compile Include="GAM\Spline1dModel.cs" />
     <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceSpectralMixture.cs" />