Free cookie consent management tool by TermsFeed Policy Generator

source: stable/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithm.cs @ 17717

Last change on this file since 17717 was 17181, checked in by swagner, 5 years ago

#2875: Merged r17180 from trunk to stable

File size: 15.4 KB
RevLine 
[12332]1#region License Information
2/* HeuristicLab
[17181]3 * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
[12332]4 * and the BEACON Center for the Study of Evolution in Action.
5 *
6 * This file is part of HeuristicLab.
7 *
8 * HeuristicLab is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * (at your option) any later version.
12 *
13 * HeuristicLab is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
20 */
21#endregion
22
[14961]23using System;
[12332]24using System.Linq;
25using System.Threading;
[17156]26using HeuristicLab.Algorithms.DataAnalysis.GradientBoostedTrees;
[12332]27using HeuristicLab.Analysis;
28using HeuristicLab.Common;
29using HeuristicLab.Core;
30using HeuristicLab.Data;
31using HeuristicLab.Optimization;
32using HeuristicLab.Parameters;
[17097]33using HEAL.Attic;
[12332]34using HeuristicLab.PluginInfrastructure;
35using HeuristicLab.Problems.DataAnalysis;
36
37namespace HeuristicLab.Algorithms.DataAnalysis {
[13977]38  [Item("Gradient Boosted Trees (GBT)", "Gradient boosted trees algorithm. Specific implementation of gradient boosting for regression trees. Friedman, J. \"Greedy Function Approximation: A Gradient Boosting Machine\", IMS 1999 Reitz Lecture.")]
[17097]39  [StorableType("8CCB55BD-4935-4868-855F-D3E5D55127AA")]
[12590]40  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 125)]
[15061]41  public class GradientBoostedTreesAlgorithm : FixedDataAnalysisAlgorithm<IRegressionProblem> {
[12332]42    #region ParameterNames
43    private const string IterationsParameterName = "Iterations";
[12632]44    private const string MaxSizeParameterName = "Maximum Tree Size";
[12332]45    private const string NuParameterName = "Nu";
46    private const string RParameterName = "R";
47    private const string MParameterName = "M";
48    private const string SeedParameterName = "Seed";
49    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
50    private const string LossFunctionParameterName = "LossFunction";
51    private const string UpdateIntervalParameterName = "UpdateInterval";
[17156]52    private const string ModelCreationParameterName = "ModelCreation";
[12332]53    #endregion
54
55    #region ParameterProperties
56    public IFixedValueParameter<IntValue> IterationsParameter {
57      get { return (IFixedValueParameter<IntValue>)Parameters[IterationsParameterName]; }
58    }
[12632]59    public IFixedValueParameter<IntValue> MaxSizeParameter {
60      get { return (IFixedValueParameter<IntValue>)Parameters[MaxSizeParameterName]; }
[12332]61    }
62    public IFixedValueParameter<DoubleValue> NuParameter {
63      get { return (IFixedValueParameter<DoubleValue>)Parameters[NuParameterName]; }
64    }
65    public IFixedValueParameter<DoubleValue> RParameter {
66      get { return (IFixedValueParameter<DoubleValue>)Parameters[RParameterName]; }
67    }
68    public IFixedValueParameter<DoubleValue> MParameter {
69      get { return (IFixedValueParameter<DoubleValue>)Parameters[MParameterName]; }
70    }
71    public IFixedValueParameter<IntValue> SeedParameter {
72      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
73    }
74    public FixedValueParameter<BoolValue> SetSeedRandomlyParameter {
75      get { return (FixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
76    }
[13184]77    public IConstrainedValueParameter<ILossFunction> LossFunctionParameter {
78      get { return (IConstrainedValueParameter<ILossFunction>)Parameters[LossFunctionParameterName]; }
[12332]79    }
80    public IFixedValueParameter<IntValue> UpdateIntervalParameter {
81      get { return (IFixedValueParameter<IntValue>)Parameters[UpdateIntervalParameterName]; }
82    }
[17156]83    private IFixedValueParameter<EnumValue<ModelCreation>> ModelCreationParameter {
84      get { return (IFixedValueParameter<EnumValue<ModelCreation>>)Parameters[ModelCreationParameterName]; }
[12373]85    }
[12332]86    #endregion
87
88    #region Properties
89    public int Iterations {
90      get { return IterationsParameter.Value.Value; }
91      set { IterationsParameter.Value.Value = value; }
92    }
93    public int Seed {
94      get { return SeedParameter.Value.Value; }
95      set { SeedParameter.Value.Value = value; }
96    }
97    public bool SetSeedRandomly {
98      get { return SetSeedRandomlyParameter.Value.Value; }
99      set { SetSeedRandomlyParameter.Value.Value = value; }
100    }
[12632]101    public int MaxSize {
102      get { return MaxSizeParameter.Value.Value; }
103      set { MaxSizeParameter.Value.Value = value; }
[12332]104    }
105    public double Nu {
106      get { return NuParameter.Value.Value; }
107      set { NuParameter.Value.Value = value; }
108    }
109    public double R {
110      get { return RParameter.Value.Value; }
111      set { RParameter.Value.Value = value; }
112    }
113    public double M {
114      get { return MParameter.Value.Value; }
115      set { MParameter.Value.Value = value; }
116    }
[17156]117    public ModelCreation ModelCreation {
118      get { return ModelCreationParameter.Value.Value; }
119      set { ModelCreationParameter.Value.Value = value; }
[12373]120    }
[12332]121    #endregion
122
123    #region ResultsProperties
124    private double ResultsBestQuality {
125      get { return ((DoubleValue)Results["Best Quality"].Value).Value; }
126      set { ((DoubleValue)Results["Best Quality"].Value).Value = value; }
127    }
128    private DataTable ResultsQualities {
129      get { return ((DataTable)Results["Qualities"].Value); }
130    }
131    #endregion
132
133    [StorableConstructor]
[17097]134    protected GradientBoostedTreesAlgorithm(StorableConstructorFlag _) : base(_) { }
[12332]135
136    protected GradientBoostedTreesAlgorithm(GradientBoostedTreesAlgorithm original, Cloner cloner)
137      : base(original, cloner) {
138    }
139
140    public override IDeepCloneable Clone(Cloner cloner) {
141      return new GradientBoostedTreesAlgorithm(this, cloner);
142    }
143
144    public GradientBoostedTreesAlgorithm() {
145      Problem = new RegressionProblem(); // default problem
146
147      Parameters.Add(new FixedValueParameter<IntValue>(IterationsParameterName, "Number of iterations (set as high as possible, adjust in combination with nu, when increasing iterations also decrease nu)", new IntValue(1000)));
148      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
149      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
[17156]150      Parameters.Add(new FixedValueParameter<IntValue>(MaxSizeParameterName, "Maximal size of the tree learned in each step (prefer smaller sizes (3 to 10) if possible)", new IntValue(10)));
[12332]151      Parameters.Add(new FixedValueParameter<DoubleValue>(RParameterName, "Ratio of training rows selected randomly in each step (0 < R <= 1)", new DoubleValue(0.5)));
152      Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName, "Ratio of variables selected randomly in each step (0 < M <= 1)", new DoubleValue(0.5)));
153      Parameters.Add(new FixedValueParameter<DoubleValue>(NuParameterName, "Learning rate nu (step size for the gradient update, should be small 0 < nu < 0.1)", new DoubleValue(0.002)));
[12373]154      Parameters.Add(new FixedValueParameter<IntValue>(UpdateIntervalParameterName, "", new IntValue(100)));
155      Parameters[UpdateIntervalParameterName].Hidden = true;
[17156]156      Parameters.Add(new FixedValueParameter<EnumValue<ModelCreation>>(ModelCreationParameterName, "Defines the results produced at the end of the run (Surrogate => Less disk space, lazy recalculation of model)", new EnumValue<ModelCreation>(ModelCreation.Model)));
157      Parameters[ModelCreationParameterName].Hidden = true;
[12332]158
[13184]159      var lossFunctions = ApplicationManager.Manager.GetInstances<ILossFunction>();
160      Parameters.Add(new ConstrainedValueParameter<ILossFunction>(LossFunctionParameterName, "The loss function", new ItemSet<ILossFunction>(lossFunctions)));
161      LossFunctionParameter.Value = LossFunctionParameter.ValidValues.First(f => f.ToString().Contains("Squared")); // squared error loss is the default
[12332]162    }
163
[13184]164    [StorableHook(HookType.AfterDeserialization)]
165    private void AfterDeserialization() {
166      // BackwardsCompatibility3.4
167      #region Backwards compatible code, remove with 3.5
[17156]168
169      #region LossFunction
[13184]170      // parameter type has been changed
171      var lossFunctionParam = Parameters[LossFunctionParameterName] as ConstrainedValueParameter<StringValue>;
172      if (lossFunctionParam != null) {
173        Parameters.Remove(LossFunctionParameterName);
174        var selectedValue = lossFunctionParam.Value; // to be restored below
[12332]175
[13184]176        var lossFunctions = ApplicationManager.Manager.GetInstances<ILossFunction>();
177        Parameters.Add(new ConstrainedValueParameter<ILossFunction>(LossFunctionParameterName, "The loss function", new ItemSet<ILossFunction>(lossFunctions)));
178        // try to restore selected value
179        var selectedLossFunction =
180          LossFunctionParameter.ValidValues.FirstOrDefault(f => f.ToString() == selectedValue.Value);
181        if (selectedLossFunction != null) {
182          LossFunctionParameter.Value = selectedLossFunction;
183        } else {
184          LossFunctionParameter.Value = LossFunctionParameter.ValidValues.First(f => f.ToString().Contains("Squared")); // default: SE
185        }
186      }
187      #endregion
[17156]188
189      #region CreateSolution
190      // parameter type has been changed
191      if (Parameters.ContainsKey("CreateSolution")) {
192        var createSolutionParam = Parameters["CreateSolution"] as FixedValueParameter<BoolValue>;
193        Parameters.Remove(createSolutionParam);
194
195        ModelCreation value = createSolutionParam.Value.Value ? ModelCreation.Model : ModelCreation.QualityOnly;
196        Parameters.Add(new FixedValueParameter<EnumValue<ModelCreation>>(ModelCreationParameterName, "Defines the results produced at the end of the run (Surrogate => Less disk space, lazy recalculation of model)", new EnumValue<ModelCreation>(value)));
197        Parameters[ModelCreationParameterName].Hidden = true;
[17157]198      } else if (!Parameters.ContainsKey(ModelCreationParameterName)) {
199        // very old version contains neither ModelCreationParameter nor CreateSolutionParameter
200        Parameters.Add(new FixedValueParameter<EnumValue<ModelCreation>>(ModelCreationParameterName, "Defines the results produced at the end of the run (Surrogate => Less disk space, lazy recalculation of model)", new EnumValue<ModelCreation>(ModelCreation.Model)));
201        Parameters[ModelCreationParameterName].Hidden = true;
[17156]202      }
203      #endregion
204      #endregion
[13184]205    }
206
[12332]207    protected override void Run(CancellationToken cancellationToken) {
208      // Set up the algorithm
[16835]209      if (SetSeedRandomly) Seed = Random.RandomSeedGenerator.GetSeed();
[12332]210
211      // Set up the results display
212      var iterations = new IntValue(0);
213      Results.Add(new Result("Iterations", iterations));
214
215      var table = new DataTable("Qualities");
216      table.Rows.Add(new DataRow("Loss (train)"));
217      table.Rows.Add(new DataRow("Loss (test)"));
[14957]218      table.Rows["Loss (train)"].VisualProperties.StartIndexZero = true;
219      table.Rows["Loss (test)"].VisualProperties.StartIndexZero = true;
220
[12332]221      Results.Add(new Result("Qualities", table));
222      var curLoss = new DoubleValue();
[12373]223      Results.Add(new Result("Loss (train)", curLoss));
[12332]224
225      // init
[12620]226      var problemData = (IRegressionProblemData)Problem.ProblemData.Clone();
[13184]227      var lossFunction = LossFunctionParameter.Value;
[12632]228      var state = GradientBoostedTreesAlgorithmStatic.CreateGbmState(problemData, lossFunction, (uint)Seed, MaxSize, R, M, Nu);
[12332]229
230      var updateInterval = UpdateIntervalParameter.Value.Value;
231      // Loop until iteration limit reached or canceled.
232      for (int i = 0; i < Iterations; i++) {
233        cancellationToken.ThrowIfCancellationRequested();
234
235        GradientBoostedTreesAlgorithmStatic.MakeStep(state);
236
237        // iteration results
238        if (i % updateInterval == 0) {
239          curLoss.Value = state.GetTrainLoss();
240          table.Rows["Loss (train)"].Values.Add(curLoss.Value);
241          table.Rows["Loss (test)"].Values.Add(state.GetTestLoss());
242          iterations.Value = i;
243        }
244      }
245
246      // final results
247      iterations.Value = Iterations;
248      curLoss.Value = state.GetTrainLoss();
249      table.Rows["Loss (train)"].Values.Add(curLoss.Value);
250      table.Rows["Loss (test)"].Values.Add(state.GetTestLoss());
251
252      // produce variable relevance
253      var orderedImpacts = state.GetVariableRelevance().Select(t => new { name = t.Key, impact = t.Value }).ToList();
254
255      var impacts = new DoubleMatrix();
256      var matrix = impacts as IStringConvertibleMatrix;
257      matrix.Rows = orderedImpacts.Count;
258      matrix.RowNames = orderedImpacts.Select(x => x.name);
259      matrix.Columns = 1;
260      matrix.ColumnNames = new string[] { "Relative variable relevance" };
261
262      int rowIdx = 0;
263      foreach (var p in orderedImpacts) {
264        matrix.SetValue(string.Format("{0:N2}", p.impact), rowIdx++, 0);
265      }
266
267      Results.Add(new Result("Variable relevance", impacts));
[12373]268      Results.Add(new Result("Loss (test)", new DoubleValue(state.GetTestLoss())));
[12332]269
270      // produce solution
[17156]271      if (ModelCreation == ModelCreation.SurrogateModel || ModelCreation == ModelCreation.Model) {
272        IRegressionModel model = state.GetModel();
[13184]273
[17156]274        if (ModelCreation == ModelCreation.SurrogateModel) {
275          model = new GradientBoostedTreesModelSurrogate((GradientBoostedTreesModel)model, problemData, (uint)Seed, lossFunction, Iterations, MaxSize, R, M, Nu);
276        }
277
[12611]278        // for logistic regression we produce a classification solution
279        if (lossFunction is LogisticRegressionLoss) {
[13184]280          var classificationModel = new DiscriminantFunctionClassificationModel(model,
[12611]281            new AccuracyMaximizationThresholdCalculator());
282          var classificationProblemData = new ClassificationProblemData(problemData.Dataset,
283            problemData.AllowedInputVariables, problemData.TargetVariable, problemData.Transformations);
[14956]284          classificationProblemData.TrainingPartition.Start = Problem.ProblemData.TrainingPartition.Start;
285          classificationProblemData.TrainingPartition.End = Problem.ProblemData.TrainingPartition.End;
286          classificationProblemData.TestPartition.Start = Problem.ProblemData.TestPartition.Start;
287          classificationProblemData.TestPartition.End = Problem.ProblemData.TestPartition.End;
[12611]288
[14957]289          classificationModel.SetThresholdsAndClassValues(new double[] { double.NegativeInfinity, 0.0 }, new[] { 0.0, 1.0 });
[14956]290
291
[13184]292          var classificationSolution = new DiscriminantFunctionClassificationSolution(classificationModel, classificationProblemData);
[12619]293          Results.Add(new Result("Solution", classificationSolution));
[12611]294        } else {
295          // otherwise we produce a regression solution
[15127]296          Results.Add(new Result("Solution", new GradientBoostedTreesSolution(model, problemData)));
[12611]297        }
[17156]298      } else if (ModelCreation == ModelCreation.QualityOnly) {
299        //Do nothing
300      } else {
301        throw new NotImplementedException("Selected parameter for CreateSolution isn't implemented yet");
[12611]302      }
[12332]303    }
304  }
305}
Note: See TracBrowser for help on using the repository browser.