Changeset 13948 for branches/HeuristicLab.RegressionSolutionGradientView/HeuristicLab.Algorithms.DataAnalysis/3.4/GBM/GradientBoostingRegressionAlgorithm.cs

Timestamp:

06/29/16 10:36:52 (8 years ago)

Author:

pfleck

Message:

#2597

• Merged recent trunk changes.
• Adapted VariablesUsedForPrediction property for RegressionSolutionTargetResponseGradientView.
• Fixed a reference (.dll to project ref).

Location:

branches/HeuristicLab.RegressionSolutionGradientView/HeuristicLab.Algorithms.DataAnalysis

Files:

• . (modified) (1 prop)
• 3.4/GBM/GradientBoostingRegressionAlgorithm.cs (modified) (14 diffs)

branches/HeuristicLab.RegressionSolutionGradientView/HeuristicLab.Algorithms.DataAnalysis/3.4/GBM/GradientBoostingRegressionAlgorithm.cs

@@ -64 +64 @@
     private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
     private const string CreateSolutionParameterName = "CreateSolution";
+    private const string StoreRunsParameterName = "StoreRuns";
     private const string RegressionAlgorithmSolutionResultParameterName = "RegressionAlgorithmResult";
 
@@ -106 +107 @@
       get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
     }
+    public IFixedValueParameter<BoolValue> StoreRunsParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[StoreRunsParameterName]; }
+    }
 
     #endregion
@@ -144 +148 @@
       get { return CreateSolutionParameter.Value.Value; }
       set { CreateSolutionParameter.Value.Value = value; }
+    }
+
+    public bool StoreRuns {
+      get { return StoreRunsParameter.Value.Value; }
+      set { StoreRunsParameter.Value.Value = value; }
     }
 
@@ -178 +187 @@
       var regressionAlgs = new ItemSet<IAlgorithm>(new IAlgorithm[] {
         new RandomForestRegression(),
-        sgp, 
+        sgp,
         mctsSymbReg
       });
@@ -206 +215 @@
         "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
       Parameters[CreateSolutionParameterName].Hidden = true;
+      Parameters.Add(new FixedValueParameter<BoolValue>(StoreRunsParameterName,
+        "Flag that indicates if the results of the individual runs should be stored for detailed analysis", new BoolValue(false)));
+      Parameters[StoreRunsParameterName].Hidden = true;
     }
 
@@ -218 +230 @@
 
       var table = new DataTable("Qualities");
-      table.Rows.Add(new DataRow("Loss (train)"));
-      table.Rows.Add(new DataRow("Loss (test)"));
+      table.Rows.Add(new DataRow("R² (train)"));
+      table.Rows.Add(new DataRow("R² (test)"));
       Results.Add(new Result("Qualities", table));
       var curLoss = new DoubleValue();
       var curTestLoss = new DoubleValue();
-      Results.Add(new Result("Loss (train)", curLoss));
-      Results.Add(new Result("Loss (test)", curTestLoss));
+      Results.Add(new Result("R² (train)", curLoss));
+      Results.Add(new Result("R² (test)", curTestLoss));
       var runCollection = new RunCollection();
-      Results.Add(new Result("Runs", runCollection));
+      if (StoreRuns)
+        Results.Add(new Result("Runs", runCollection));
 
       // init
       var problemData = Problem.ProblemData;
-      var targetVarName = Problem.ProblemData.TargetVariable;
+      var targetVarName = problemData.TargetVariable;
       var activeVariables = problemData.AllowedInputVariables.Concat(new string[] { problemData.TargetVariable });
       var modifiableDataset = new ModifiableDataset(
@@ -273 +286 @@
           IRegressionModel model;
           IRun run;
+
           // try to find a model. The algorithm might fail to produce a model. In this case we just retry until the iterations are exhausted
-          if (TryExecute(alg, RegressionAlgorithmResult, out model, out run)) {
+          if (TryExecute(alg, rand.Next(), RegressionAlgorithmResult, out model, out run)) {
             int row = 0;
             // update predictions for training and test
@@ -303 +317 @@
           }
 
-          runCollection.Add(run);
-          table.Rows["Loss (train)"].Values.Add(curLoss.Value);
-          table.Rows["Loss (test)"].Values.Add(curTestLoss.Value);
+          if (StoreRuns)
+            runCollection.Add(run);
+          table.Rows["R² (train)"].Values.Add(curLoss.Value);
+          table.Rows["R² (test)"].Values.Add(curTestLoss.Value);
           iterations.Value = i + 1;
         }
@@ -317 +332 @@
           // just produce an ensemble solution for now (TODO: correct scaling or linear regression for ensemble model weights)
 
-          var ensembleModel = new RegressionEnsembleModel(models) { AverageModelEstimates = false };
-          var ensembleSolution = ensembleModel.CreateRegressionSolution((IRegressionProblemData)problemData.Clone());
+          var ensembleSolution = CreateEnsembleSolution(models, (IRegressionProblemData)problemData.Clone());
           Results.Add(new Result("EnsembleSolution", ensembleSolution));
         }
@@ -326 +340 @@
         alg.Prepare(true);
       }
+    }
+
+    private static IRegressionEnsembleSolution CreateEnsembleSolution(List<IRegressionModel> models,
+      IRegressionProblemData problemData) {
+      var rows = problemData.TrainingPartition.Size;
+      var features = models.Count;
+      double[,] inputMatrix = new double[rows, features + 1];
+      //add model estimates
+      for (int m = 0; m < models.Count; m++) {
+        var model = models[m];
+        var estimates = model.GetEstimatedValues(problemData.Dataset, problemData.TrainingIndices);
+        int estimatesCounter = 0;
+        foreach (var estimate in estimates) {
+          inputMatrix[estimatesCounter, m] = estimate;
+          estimatesCounter++;
+        }
+      }
+
+      //add target
+      var targets = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+      int targetCounter = 0;
+      foreach (var target in targets) {
+        inputMatrix[targetCounter, models.Count] = target;
+        targetCounter++;
+      }
+
+      alglib.linearmodel lm = new alglib.linearmodel();
+      alglib.lrreport ar = new alglib.lrreport();
+      double[] coefficients;
+      int retVal = 1;
+      alglib.lrbuildz(inputMatrix, rows, features, out retVal, out lm, out ar);
+      if (retVal != 1) throw new ArgumentException("Error in calculation of linear regression solution");
+
+      alglib.lrunpack(lm, out coefficients, out features);
+
+      var ensembleModel = new RegressionEnsembleModel(models, coefficients.Take(models.Count)) { AverageModelEstimates = false };
+      var ensembleSolution = (IRegressionEnsembleSolution)ensembleModel.CreateRegressionSolution(problemData);
+      return ensembleSolution;
     }
 
@@ -391 +443 @@
       progRootNode.AddSubtree(startNode);
       var t = new SymbolicExpressionTree(progRootNode);
-      var combinedModel = new SymbolicRegressionModel(t, interpreter, lowerLimit, upperLimit);
+      var combinedModel = new SymbolicRegressionModel(problemData.TargetVariable, t, interpreter, lowerLimit, upperLimit);
       var sol = new SymbolicRegressionSolution(combinedModel, problemData);
       return sol;
@@ -405 +457 @@
     }
 
-    private static bool TryExecute(IAlgorithm alg, string regressionAlgorithmResultName, out IRegressionModel model, out IRun run) {
+    private static bool TryExecute(IAlgorithm alg, int seed, string regressionAlgorithmResultName, out IRegressionModel model, out IRun run) {
       model = null;
+      SetSeed(alg, seed);
       using (var wh = new AutoResetEvent(false)) {
-        EventHandler<EventArgs<Exception>> handler = (sender, args) => wh.Set();
+        Exception ex = null;
+        EventHandler<EventArgs<Exception>> handler = (sender, args) => {
+          ex = args.Value;
+          wh.Set();
+        };
         EventHandler handler2 = (sender, args) => wh.Set();
         alg.ExceptionOccurred += handler;
@@ -417 +474 @@
           wh.WaitOne();
 
+          if (ex != null) throw new AggregateException(ex);
           run = alg.Runs.Last();
+          alg.Runs.Clear();
           var sols = alg.Results.Select(r => r.Value).OfType<IRegressionSolution>();
           if (!sols.Any()) return false;
@@ -444 +503 @@
       return model != null;
     }
+
+    private static void SetSeed(IAlgorithm alg, int seed) {
+      // no common interface for algs that use a PRNG -> use naming convention to set seed
+      var paramItem = alg as IParameterizedItem;
+
+      if (paramItem.Parameters.ContainsKey("SetSeedRandomly")) {
+        ((BoolValue)paramItem.Parameters["SetSeedRandomly"].ActualValue).Value = false;
+        ((IntValue)paramItem.Parameters["Seed"].ActualValue).Value = seed;
+      } else {
+        throw new ArgumentException("Base learner does not have a seed parameter (algorithm {0})", alg.Name);
+      }
+
+    }
   }
 }