Changeset 13977

Timestamp:

07/02/16 09:08:09 (8 years ago)

Author:

gkronber

Message:

#1795: merged r13646,13653,13655,13699,13703,13707,13889,13898,13917 from trunk to stable

Location:

stable

Files:

• . (modified) (1 prop)
• HeuristicLab.Algorithms.DataAnalysis (modified) (1 prop)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GBM (copied) (copied from trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GBM)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GBM/GradientBoostingRegressionAlgorithm.cs (modified) (15 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithm.cs (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj (modified) (3 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame (modified) (2 diffs)

stable/HeuristicLab.Algorithms.DataAnalysis/3.4/GBM/GradientBoostingRegressionAlgorithm.cs

@@ -37 +37 @@
 using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
 using HeuristicLab.Random;
+using HeuristicLab.Selection;
 
 namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression {
@@ -63 +64 @@
     private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
     private const string CreateSolutionParameterName = "CreateSolution";
+    private const string StoreRunsParameterName = "StoreRuns";
     private const string RegressionAlgorithmSolutionResultParameterName = "RegressionAlgorithmResult";
 
@@ -105 +107 @@
       get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
     }
+    public IFixedValueParameter<BoolValue> StoreRunsParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[StoreRunsParameterName]; }
+    }
 
     #endregion
@@ -145 +150 @@
     }
 
+    public bool StoreRuns {
+      get { return StoreRunsParameter.Value.Value; }
+      set { StoreRunsParameter.Value.Value = value; }
+    }
+
     public IAlgorithm RegressionAlgorithm {
       get { return RegressionAlgorithmParameter.Value; }
@@ -172 +182 @@
       Problem = new RegressionProblem(); // default problem
       var mctsSymbReg = new MctsSymbolicRegressionAlgorithm();
-      // var sgp = CreateSGP();
+      mctsSymbReg.Iterations = 10000;
+      mctsSymbReg.StoreAlgorithmInEachRun = false;
+      var sgp = CreateOSGP();
       var regressionAlgs = new ItemSet<IAlgorithm>(new IAlgorithm[] {
-        new LinearRegression(), new RandomForestRegression(), new NearestNeighbourRegression(),
-        // sgp, 
+        new RandomForestRegression(),
+        sgp,
         mctsSymbReg
       });
@@ -203 +215 @@
         "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
       Parameters[CreateSolutionParameterName].Hidden = true;
+      Parameters.Add(new FixedValueParameter<BoolValue>(StoreRunsParameterName,
+        "Flag that indicates if the results of the individual runs should be stored for detailed analysis", new BoolValue(false)));
+      Parameters[StoreRunsParameterName].Hidden = true;
     }
 
@@ -215 +230 @@
 
       var table = new DataTable("Qualities");
-      table.Rows.Add(new DataRow("Loss (train)"));
-      table.Rows.Add(new DataRow("Loss (test)"));
+      table.Rows.Add(new DataRow("R² (train)"));
+      table.Rows.Add(new DataRow("R² (test)"));
       Results.Add(new Result("Qualities", table));
       var curLoss = new DoubleValue();
       var curTestLoss = new DoubleValue();
-      Results.Add(new Result("Loss (train)", curLoss));
-      Results.Add(new Result("Loss (test)", curTestLoss));
+      Results.Add(new Result("R² (train)", curLoss));
+      Results.Add(new Result("R² (test)", curTestLoss));
       var runCollection = new RunCollection();
-      Results.Add(new Result("Runs", runCollection));
+      if (StoreRuns)
+        Results.Add(new Result("Runs", runCollection));
 
       // init
       var problemData = Problem.ProblemData;
-      var targetVarName = Problem.ProblemData.TargetVariable;
+      var targetVarName = problemData.TargetVariable;
+      var activeVariables = problemData.AllowedInputVariables.Concat(new string[] { problemData.TargetVariable });
       var modifiableDataset = new ModifiableDataset(
-        problemData.Dataset.VariableNames,
-        problemData.Dataset.VariableNames.Select(v => problemData.Dataset.GetDoubleValues(v).ToList()));
+        activeVariables,
+        activeVariables.Select(v => problemData.Dataset.GetDoubleValues(v).ToList()));
 
       var trainingRows = problemData.TrainingIndices;
@@ -269 +286 @@
           IRegressionModel model;
           IRun run;
+
           // try to find a model. The algorithm might fail to produce a model. In this case we just retry until the iterations are exhausted
-          if (TryExecute(alg, RegressionAlgorithmResult, out model, out run)) {
+          if (TryExecute(alg, rand.Next(), RegressionAlgorithmResult, out model, out run)) {
             int row = 0;
             // update predictions for training and test
@@ -299 +317 @@
           }
 
-          runCollection.Add(run);
-          table.Rows["Loss (train)"].Values.Add(curLoss.Value);
-          table.Rows["Loss (test)"].Values.Add(curTestLoss.Value);
+          if (StoreRuns)
+            runCollection.Add(run);
+          table.Rows["R² (train)"].Values.Add(curLoss.Value);
+          table.Rows["R² (test)"].Values.Add(curTestLoss.Value);
           iterations.Value = i + 1;
         }
@@ -312 +331 @@
           }
           // just produce an ensemble solution for now (TODO: correct scaling or linear regression for ensemble model weights)
-          Results.Add(new Result("EnsembleSolution", new RegressionEnsembleSolution(models, (IRegressionProblemData)problemData.Clone())));
+
+          var ensembleSolution = CreateEnsembleSolution(models, (IRegressionProblemData)problemData.Clone());
+          Results.Add(new Result("EnsembleSolution", ensembleSolution));
         }
-      } finally {
+      }
+      finally {
         // reset everything
         alg.Prepare(true);
@@ -320 +342 @@
     }
 
-    // this is probably slow as hell
+    private static IRegressionEnsembleSolution CreateEnsembleSolution(List<IRegressionModel> models,
+      IRegressionProblemData problemData) {
+      var rows = problemData.TrainingPartition.Size;
+      var features = models.Count;
+      double[,] inputMatrix = new double[rows, features + 1];
+      //add model estimates
+      for (int m = 0; m < models.Count; m++) {
+        var model = models[m];
+        var estimates = model.GetEstimatedValues(problemData.Dataset, problemData.TrainingIndices);
+        int estimatesCounter = 0;
+        foreach (var estimate in estimates) {
+          inputMatrix[estimatesCounter, m] = estimate;
+          estimatesCounter++;
+        }
+      }
+
+      //add target
+      var targets = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+      int targetCounter = 0;
+      foreach (var target in targets) {
+        inputMatrix[targetCounter, models.Count] = target;
+        targetCounter++;
+      }
+
+      alglib.linearmodel lm = new alglib.linearmodel();
+      alglib.lrreport ar = new alglib.lrreport();
+      double[] coefficients;
+      int retVal = 1;
+      alglib.lrbuildz(inputMatrix, rows, features, out retVal, out lm, out ar);
+      if (retVal != 1) throw new ArgumentException("Error in calculation of linear regression solution");
+
+      alglib.lrunpack(lm, out coefficients, out features);
+
+      var ensembleModel = new RegressionEnsembleModel(models, coefficients.Take(models.Count)) { AverageModelEstimates = false };
+      var ensembleSolution = ensembleModel.CreateRegressionSolution(problemData);
+      return ensembleSolution;
+    }
+
+
+    private IAlgorithm CreateOSGP() {
+      // configure strict osgp
+      var alg = new OffspringSelectionGeneticAlgorithm.OffspringSelectionGeneticAlgorithm();
+      var prob = new SymbolicRegressionSingleObjectiveProblem();
+      prob.MaximumSymbolicExpressionTreeDepth.Value = 7;
+      prob.MaximumSymbolicExpressionTreeLength.Value = 15;
+      alg.Problem = prob;
+      alg.SuccessRatio.Value = 1.0;
+      alg.ComparisonFactorLowerBound.Value = 1.0;
+      alg.ComparisonFactorUpperBound.Value = 1.0;
+      alg.MutationProbability.Value = 0.15;
+      alg.PopulationSize.Value = 200;
+      alg.MaximumSelectionPressure.Value = 100;
+      alg.MaximumEvaluatedSolutions.Value = 20000;
+      alg.SelectorParameter.Value = alg.SelectorParameter.ValidValues.OfType<GenderSpecificSelector>().First();
+      alg.MutatorParameter.Value = alg.MutatorParameter.ValidValues.OfType<MultiSymbolicExpressionTreeManipulator>().First();
+      alg.StoreAlgorithmInEachRun = false;
+      return alg;
+    }
+
     private void SampleTrainingData(MersenneTwister rand, ModifiableDataset ds, int rRows,
       IDataset sourceDs, double[] curTarget, string targetVarName, IEnumerable<int> trainingIndices) {
@@ -374 +454 @@
         prob.ProblemDataParameter.Value = problemData;
         return true;
-      } else if (alg.Problem != null) {
-        // a problem is set and it is not compatible
-        return false;
-      } else {
-        try {
-          // we try to use a symbolic regression problem (works for simple regression algs and GP)
-          alg.Problem = new SymbolicRegressionSingleObjectiveProblem();
-        } catch (Exception) {
-          return false;
-        }
-        return true;
-      }
-    }
-
-    private static bool TryExecute(IAlgorithm alg, string regressionAlgorithmResultName, out IRegressionModel model, out IRun run) {
+      } else return false;
+    }
+
+    private static bool TryExecute(IAlgorithm alg, int seed, string regressionAlgorithmResultName, out IRegressionModel model, out IRun run) {
       model = null;
+      SetSeed(alg, seed);
       using (var wh = new AutoResetEvent(false)) {
-        EventHandler<EventArgs<Exception>> handler = (sender, args) => wh.Set();
+        Exception ex = null;
+        EventHandler<EventArgs<Exception>> handler = (sender, args) => {
+          ex = args.Value;
+          wh.Set();
+        };
         EventHandler handler2 = (sender, args) => wh.Set();
         alg.ExceptionOccurred += handler;
@@ -400 +474 @@
           wh.WaitOne();
 
+          if (ex != null) throw new AggregateException(ex);
           run = alg.Runs.Last();
+          alg.Runs.Clear();
           var sols = alg.Results.Select(r => r.Value).OfType<IRegressionSolution>();
           if (!sols.Any()) return false;
@@ -419 +495 @@
             model = sol.Model;
           }
-        } finally {
+        }
+        finally {
           alg.ExceptionOccurred -= handler;
           alg.Stopped -= handler2;
@@ -426 +503 @@
       return model != null;
     }
+
+    private static void SetSeed(IAlgorithm alg, int seed) {
+      // no common interface for algs that use a PRNG -> use naming convention to set seed
+      var paramItem = alg as IParameterizedItem;
+
+      if (paramItem.Parameters.ContainsKey("SetSeedRandomly")) {
+        ((BoolValue)paramItem.Parameters["SetSeedRandomly"].ActualValue).Value = false;
+        ((IntValue)paramItem.Parameters["Seed"].ActualValue).Value = seed;
+      } else {
+        throw new ArgumentException("Base learner does not have a seed parameter (algorithm {0})", alg.Name);
+      }
+
+    }
   }
 }

stable/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithm.cs

@@ -35 +35 @@
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
-  [Item("Gradient Boosted Trees (GBT)", "Gradient boosted trees algorithm. Friedman, J. \"Greedy Function Approximation: A Gradient Boosting Machine\", IMS 1999 Reitz Lecture.")]
+  [Item("Gradient Boosted Trees (GBT)", "Gradient boosted trees algorithm. Specific implementation of gradient boosting for regression trees. Friedman, J. \"Greedy Function Approximation: A Gradient Boosting Machine\", IMS 1999 Reitz Lecture.")]
   [StorableClass]
   [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 125)]

stable/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj

@@ -201 +201 @@
     <Compile Include="GaussianProcess\GaussianProcessRegressionSolution.cs" />
     <Compile Include="GaussianProcess\ICovarianceFunction.cs" />
+    <Compile Include="GBM\GradientBoostingRegressionAlgorithm.cs" />
     <Compile Include="GradientBoostedTrees\IGradientBoostedTreesModel.cs" />
     <Compile Include="GradientBoostedTrees\GradientBoostedTreesModelSurrogate.cs" />
@@ -309 +310 @@
       <Private>False</Private>
     </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm\3.3\HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm-3.3.csproj">
+      <Project>{F409DD9E-1E9C-4EB1-AA3A-9F6E987C6E58}</Project>
+      <Name>HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm-3.3</Name>
+    </ProjectReference>
     <ProjectReference Include="..\..\HeuristicLab.Analysis\3.3\HeuristicLab.Analysis-3.3.csproj">
       <Project>{887425B4-4348-49ED-A457-B7D2C26DDBF9}</Project>
@@ -407 +412 @@
       <Name>HeuristicLab.Random-3.3</Name>
       <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Selection\3.3\HeuristicLab.Selection-3.3.csproj">
+      <Project>{2C36CD4F-E5F5-43A4-801A-201EA895FE17}</Project>
+      <Name>HeuristicLab.Selection-3.3</Name>
     </ProjectReference>
   </ItemGroup>

stable/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame

@@ -29 +29 @@
   [PluginFile("HeuristicLab.Algorithms.DataAnalysis-3.4.dll", PluginFileType.Assembly)]
   [PluginDependency("HeuristicLab.ALGLIB", "3.7.0")]
+  [PluginDependency("HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm", "3.3")] // for GBM
   [PluginDependency("HeuristicLab.Algorithms.GradientDescent", "3.3")]
   [PluginDependency("HeuristicLab.Analysis", "3.3")]
@@ -50 +51 @@
   [PluginDependency("HeuristicLab.LibSVM", "3.12")]
   [PluginDependency("HeuristicLab.Random", "3.3")]
+  [PluginDependency("HeuristicLab.Selection", "3.3")]
   public class HeuristicLabAlgorithmsDataAnalysisPlugin : PluginBase {
   }