Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithm.cs @ 16685

Last change on this file since 16685 was 16565, checked in by gkronber, 6 years ago

#2520: merged changes from PersistenceOverhaul branch (r16451:16564) into trunk

File size: 13.4 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 * and the BEACON Center for the Study of Evolution in Action.
5 *
6 * This file is part of HeuristicLab.
7 *
8 * HeuristicLab is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * (at your option) any later version.
12 *
13 * HeuristicLab is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
20 */
21#endregion
22
23using System.Linq;
24using System.Threading;
25using HeuristicLab.Analysis;
26using HeuristicLab.Common;
27using HeuristicLab.Core;
28using HeuristicLab.Data;
29using HeuristicLab.Optimization;
30using HeuristicLab.Parameters;
31using HEAL.Attic;
32using HeuristicLab.PluginInfrastructure;
33using HeuristicLab.Problems.DataAnalysis;
34
35namespace HeuristicLab.Algorithms.DataAnalysis {
36  [Item("Gradient Boosted Trees (GBT)", "Gradient boosted trees algorithm. Specific implementation of gradient boosting for regression trees. Friedman, J. \"Greedy Function Approximation: A Gradient Boosting Machine\", IMS 1999 Reitz Lecture.")]
37  [StorableType("8CCB55BD-4935-4868-855F-D3E5D55127AA")]
38  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 125)]
39  public class GradientBoostedTreesAlgorithm : FixedDataAnalysisAlgorithm<IRegressionProblem> {
40    #region ParameterNames
41    private const string IterationsParameterName = "Iterations";
42    private const string MaxSizeParameterName = "Maximum Tree Size";
43    private const string NuParameterName = "Nu";
44    private const string RParameterName = "R";
45    private const string MParameterName = "M";
46    private const string SeedParameterName = "Seed";
47    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
48    private const string LossFunctionParameterName = "LossFunction";
49    private const string UpdateIntervalParameterName = "UpdateInterval";
50    private const string CreateSolutionParameterName = "CreateSolution";
51    #endregion
52
53    #region ParameterProperties
54    public IFixedValueParameter<IntValue> IterationsParameter {
55      get { return (IFixedValueParameter<IntValue>)Parameters[IterationsParameterName]; }
56    }
57    public IFixedValueParameter<IntValue> MaxSizeParameter {
58      get { return (IFixedValueParameter<IntValue>)Parameters[MaxSizeParameterName]; }
59    }
60    public IFixedValueParameter<DoubleValue> NuParameter {
61      get { return (IFixedValueParameter<DoubleValue>)Parameters[NuParameterName]; }
62    }
63    public IFixedValueParameter<DoubleValue> RParameter {
64      get { return (IFixedValueParameter<DoubleValue>)Parameters[RParameterName]; }
65    }
66    public IFixedValueParameter<DoubleValue> MParameter {
67      get { return (IFixedValueParameter<DoubleValue>)Parameters[MParameterName]; }
68    }
69    public IFixedValueParameter<IntValue> SeedParameter {
70      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
71    }
72    public FixedValueParameter<BoolValue> SetSeedRandomlyParameter {
73      get { return (FixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
74    }
75    public IConstrainedValueParameter<ILossFunction> LossFunctionParameter {
76      get { return (IConstrainedValueParameter<ILossFunction>)Parameters[LossFunctionParameterName]; }
77    }
78    public IFixedValueParameter<IntValue> UpdateIntervalParameter {
79      get { return (IFixedValueParameter<IntValue>)Parameters[UpdateIntervalParameterName]; }
80    }
81    public IFixedValueParameter<BoolValue> CreateSolutionParameter {
82      get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
83    }
84    #endregion
85
86    #region Properties
87    public int Iterations {
88      get { return IterationsParameter.Value.Value; }
89      set { IterationsParameter.Value.Value = value; }
90    }
91    public int Seed {
92      get { return SeedParameter.Value.Value; }
93      set { SeedParameter.Value.Value = value; }
94    }
95    public bool SetSeedRandomly {
96      get { return SetSeedRandomlyParameter.Value.Value; }
97      set { SetSeedRandomlyParameter.Value.Value = value; }
98    }
99    public int MaxSize {
100      get { return MaxSizeParameter.Value.Value; }
101      set { MaxSizeParameter.Value.Value = value; }
102    }
103    public double Nu {
104      get { return NuParameter.Value.Value; }
105      set { NuParameter.Value.Value = value; }
106    }
107    public double R {
108      get { return RParameter.Value.Value; }
109      set { RParameter.Value.Value = value; }
110    }
111    public double M {
112      get { return MParameter.Value.Value; }
113      set { MParameter.Value.Value = value; }
114    }
115    public bool CreateSolution {
116      get { return CreateSolutionParameter.Value.Value; }
117      set { CreateSolutionParameter.Value.Value = value; }
118    }
119    #endregion
120
121    #region ResultsProperties
122    private double ResultsBestQuality {
123      get { return ((DoubleValue)Results["Best Quality"].Value).Value; }
124      set { ((DoubleValue)Results["Best Quality"].Value).Value = value; }
125    }
126    private DataTable ResultsQualities {
127      get { return ((DataTable)Results["Qualities"].Value); }
128    }
129    #endregion
130
131    [StorableConstructor]
132    protected GradientBoostedTreesAlgorithm(StorableConstructorFlag _) : base(_) { }
133
134    protected GradientBoostedTreesAlgorithm(GradientBoostedTreesAlgorithm original, Cloner cloner)
135      : base(original, cloner) {
136    }
137
138    public override IDeepCloneable Clone(Cloner cloner) {
139      return new GradientBoostedTreesAlgorithm(this, cloner);
140    }
141
142    public GradientBoostedTreesAlgorithm() {
143      Problem = new RegressionProblem(); // default problem
144
145      Parameters.Add(new FixedValueParameter<IntValue>(IterationsParameterName, "Number of iterations (set as high as possible, adjust in combination with nu, when increasing iterations also decrease nu)", new IntValue(1000)));
146      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
147      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
148      Parameters.Add(new FixedValueParameter<IntValue>(MaxSizeParameterName, "Maximal size of the tree learned in each step (prefer smaller sizes if possible)", new IntValue(10)));
149      Parameters.Add(new FixedValueParameter<DoubleValue>(RParameterName, "Ratio of training rows selected randomly in each step (0 < R <= 1)", new DoubleValue(0.5)));
150      Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName, "Ratio of variables selected randomly in each step (0 < M <= 1)", new DoubleValue(0.5)));
151      Parameters.Add(new FixedValueParameter<DoubleValue>(NuParameterName, "Learning rate nu (step size for the gradient update, should be small 0 < nu < 0.1)", new DoubleValue(0.002)));
152      Parameters.Add(new FixedValueParameter<IntValue>(UpdateIntervalParameterName, "", new IntValue(100)));
153      Parameters[UpdateIntervalParameterName].Hidden = true;
154      Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
155      Parameters[CreateSolutionParameterName].Hidden = true;
156
157      var lossFunctions = ApplicationManager.Manager.GetInstances<ILossFunction>();
158      Parameters.Add(new ConstrainedValueParameter<ILossFunction>(LossFunctionParameterName, "The loss function", new ItemSet<ILossFunction>(lossFunctions)));
159      LossFunctionParameter.Value = LossFunctionParameter.ValidValues.First(f => f.ToString().Contains("Squared")); // squared error loss is the default
160    }
161
162    [StorableHook(HookType.AfterDeserialization)]
163    private void AfterDeserialization() {
164      // BackwardsCompatibility3.4
165      #region Backwards compatible code, remove with 3.5
166      // parameter type has been changed
167      var lossFunctionParam = Parameters[LossFunctionParameterName] as ConstrainedValueParameter<StringValue>;
168      if (lossFunctionParam != null) {
169        Parameters.Remove(LossFunctionParameterName);
170        var selectedValue = lossFunctionParam.Value; // to be restored below
171
172        var lossFunctions = ApplicationManager.Manager.GetInstances<ILossFunction>();
173        Parameters.Add(new ConstrainedValueParameter<ILossFunction>(LossFunctionParameterName, "The loss function", new ItemSet<ILossFunction>(lossFunctions)));
174        // try to restore selected value
175        var selectedLossFunction =
176          LossFunctionParameter.ValidValues.FirstOrDefault(f => f.ToString() == selectedValue.Value);
177        if (selectedLossFunction != null) {
178          LossFunctionParameter.Value = selectedLossFunction;
179        } else {
180          LossFunctionParameter.Value = LossFunctionParameter.ValidValues.First(f => f.ToString().Contains("Squared")); // default: SE
181        }
182      }
183      #endregion
184    }
185
186    protected override void Run(CancellationToken cancellationToken) {
187      // Set up the algorithm
188      if (SetSeedRandomly) Seed = Random.RandomSeedGenerator.GetSeed();
189
190      // Set up the results display
191      var iterations = new IntValue(0);
192      Results.Add(new Result("Iterations", iterations));
193
194      var table = new DataTable("Qualities");
195      table.Rows.Add(new DataRow("Loss (train)"));
196      table.Rows.Add(new DataRow("Loss (test)"));
197      table.Rows["Loss (train)"].VisualProperties.StartIndexZero = true;
198      table.Rows["Loss (test)"].VisualProperties.StartIndexZero = true;
199
200      Results.Add(new Result("Qualities", table));
201      var curLoss = new DoubleValue();
202      Results.Add(new Result("Loss (train)", curLoss));
203
204      // init
205      var problemData = (IRegressionProblemData)Problem.ProblemData.Clone();
206      var lossFunction = LossFunctionParameter.Value;
207      var state = GradientBoostedTreesAlgorithmStatic.CreateGbmState(problemData, lossFunction, (uint)Seed, MaxSize, R, M, Nu);
208
209      var updateInterval = UpdateIntervalParameter.Value.Value;
210      // Loop until iteration limit reached or canceled.
211      for (int i = 0; i < Iterations; i++) {
212        cancellationToken.ThrowIfCancellationRequested();
213
214        GradientBoostedTreesAlgorithmStatic.MakeStep(state);
215
216        // iteration results
217        if (i % updateInterval == 0) {
218          curLoss.Value = state.GetTrainLoss();
219          table.Rows["Loss (train)"].Values.Add(curLoss.Value);
220          table.Rows["Loss (test)"].Values.Add(state.GetTestLoss());
221          iterations.Value = i;
222        }
223      }
224
225      // final results
226      iterations.Value = Iterations;
227      curLoss.Value = state.GetTrainLoss();
228      table.Rows["Loss (train)"].Values.Add(curLoss.Value);
229      table.Rows["Loss (test)"].Values.Add(state.GetTestLoss());
230
231      // produce variable relevance
232      var orderedImpacts = state.GetVariableRelevance().Select(t => new { name = t.Key, impact = t.Value }).ToList();
233
234      var impacts = new DoubleMatrix();
235      var matrix = impacts as IStringConvertibleMatrix;
236      matrix.Rows = orderedImpacts.Count;
237      matrix.RowNames = orderedImpacts.Select(x => x.name);
238      matrix.Columns = 1;
239      matrix.ColumnNames = new string[] { "Relative variable relevance" };
240
241      int rowIdx = 0;
242      foreach (var p in orderedImpacts) {
243        matrix.SetValue(string.Format("{0:N2}", p.impact), rowIdx++, 0);
244      }
245
246      Results.Add(new Result("Variable relevance", impacts));
247      Results.Add(new Result("Loss (test)", new DoubleValue(state.GetTestLoss())));
248
249      // produce solution
250      if (CreateSolution) {
251        var model = state.GetModel();
252
253        // for logistic regression we produce a classification solution
254        if (lossFunction is LogisticRegressionLoss) {
255          var classificationModel = new DiscriminantFunctionClassificationModel(model,
256            new AccuracyMaximizationThresholdCalculator());
257          var classificationProblemData = new ClassificationProblemData(problemData.Dataset,
258            problemData.AllowedInputVariables, problemData.TargetVariable, problemData.Transformations);
259          classificationProblemData.TrainingPartition.Start = Problem.ProblemData.TrainingPartition.Start;
260          classificationProblemData.TrainingPartition.End = Problem.ProblemData.TrainingPartition.End;
261          classificationProblemData.TestPartition.Start = Problem.ProblemData.TestPartition.Start;
262          classificationProblemData.TestPartition.End = Problem.ProblemData.TestPartition.End;
263
264          classificationModel.SetThresholdsAndClassValues(new double[] { double.NegativeInfinity, 0.0 }, new[] { 0.0, 1.0 });
265
266
267          var classificationSolution = new DiscriminantFunctionClassificationSolution(classificationModel, classificationProblemData);
268          Results.Add(new Result("Solution", classificationSolution));
269        } else {
270          // otherwise we produce a regression solution
271          Results.Add(new Result("Solution", new GradientBoostedTreesSolution(model, problemData)));
272        }
273      }
274    }
275  }
276}
Note: See TracBrowser for help on using the repository browser.