Free cookie consent management tool by TermsFeed Policy Generator

source: branches/2883_GBTModelStorage/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithm.cs @ 15679

Last change on this file since 15679 was 15679, checked in by fholzing, 6 years ago

#2883: Removed backwardscompatibility and changed the level of the decision if surrogate or not into the algorithm (so one level up)

File size: 14.9 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 * and the BEACON Center for the Study of Evolution in Action.
5 *
6 * This file is part of HeuristicLab.
7 *
8 * HeuristicLab is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * (at your option) any later version.
12 *
13 * HeuristicLab is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
20 */
21#endregion
22
23using System;
24using System.Linq;
25using System.Threading;
26using HeuristicLab.Algorithms.DataAnalysis.GradientBoostedTrees;
27using HeuristicLab.Analysis;
28using HeuristicLab.Common;
29using HeuristicLab.Core;
30using HeuristicLab.Data;
31using HeuristicLab.Optimization;
32using HeuristicLab.Parameters;
33using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
34using HeuristicLab.PluginInfrastructure;
35using HeuristicLab.Problems.DataAnalysis;
36
37namespace HeuristicLab.Algorithms.DataAnalysis {
38  [Item("Gradient Boosted Trees (GBT)", "Gradient boosted trees algorithm. Specific implementation of gradient boosting for regression trees. Friedman, J. \"Greedy Function Approximation: A Gradient Boosting Machine\", IMS 1999 Reitz Lecture.")]
39  [StorableClass]
40  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 125)]
41  public class GradientBoostedTreesAlgorithm : FixedDataAnalysisAlgorithm<IRegressionProblem> {
42    #region ParameterNames
43    private const string IterationsParameterName = "Iterations";
44    private const string MaxSizeParameterName = "Maximum Tree Size";
45    private const string NuParameterName = "Nu";
46    private const string RParameterName = "R";
47    private const string MParameterName = "M";
48    private const string SeedParameterName = "Seed";
49    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
50    private const string LossFunctionParameterName = "LossFunction";
51    private const string UpdateIntervalParameterName = "UpdateInterval";
52    private const string CreateSolutionParameterName = "CreateSolution";
53    #endregion
54
55    #region ParameterProperties
56    public IFixedValueParameter<IntValue> IterationsParameter
57    {
58      get { return (IFixedValueParameter<IntValue>)Parameters[IterationsParameterName]; }
59    }
60    public IFixedValueParameter<IntValue> MaxSizeParameter
61    {
62      get { return (IFixedValueParameter<IntValue>)Parameters[MaxSizeParameterName]; }
63    }
64    public IFixedValueParameter<DoubleValue> NuParameter
65    {
66      get { return (IFixedValueParameter<DoubleValue>)Parameters[NuParameterName]; }
67    }
68    public IFixedValueParameter<DoubleValue> RParameter
69    {
70      get { return (IFixedValueParameter<DoubleValue>)Parameters[RParameterName]; }
71    }
72    public IFixedValueParameter<DoubleValue> MParameter
73    {
74      get { return (IFixedValueParameter<DoubleValue>)Parameters[MParameterName]; }
75    }
76    public IFixedValueParameter<IntValue> SeedParameter
77    {
78      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
79    }
80    public FixedValueParameter<BoolValue> SetSeedRandomlyParameter
81    {
82      get { return (FixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
83    }
84    public IConstrainedValueParameter<ILossFunction> LossFunctionParameter
85    {
86      get { return (IConstrainedValueParameter<ILossFunction>)Parameters[LossFunctionParameterName]; }
87    }
88    public IFixedValueParameter<IntValue> UpdateIntervalParameter
89    {
90      get { return (IFixedValueParameter<IntValue>)Parameters[UpdateIntervalParameterName]; }
91    }
92    private IFixedValueParameter<EnumValue<ModelStorage>> CreateSolutionParameter
93    {
94      get { return (IFixedValueParameter<EnumValue<ModelStorage>>)Parameters[CreateSolutionParameterName]; }
95    }
96    #endregion
97
98    #region Properties
99    public int Iterations
100    {
101      get { return IterationsParameter.Value.Value; }
102      set { IterationsParameter.Value.Value = value; }
103    }
104    public int Seed
105    {
106      get { return SeedParameter.Value.Value; }
107      set { SeedParameter.Value.Value = value; }
108    }
109    public bool SetSeedRandomly
110    {
111      get { return SetSeedRandomlyParameter.Value.Value; }
112      set { SetSeedRandomlyParameter.Value.Value = value; }
113    }
114    public int MaxSize
115    {
116      get { return MaxSizeParameter.Value.Value; }
117      set { MaxSizeParameter.Value.Value = value; }
118    }
119    public double Nu
120    {
121      get { return NuParameter.Value.Value; }
122      set { NuParameter.Value.Value = value; }
123    }
124    public double R
125    {
126      get { return RParameter.Value.Value; }
127      set { RParameter.Value.Value = value; }
128    }
129    public double M
130    {
131      get { return MParameter.Value.Value; }
132      set { MParameter.Value.Value = value; }
133    }
134    public ModelStorage CreateSolution
135    {
136      get { return CreateSolutionParameter.Value.Value; }
137      set { CreateSolutionParameter.Value.Value = value; }
138    }
139    #endregion
140
141    #region ResultsProperties
142    private double ResultsBestQuality
143    {
144      get { return ((DoubleValue)Results["Best Quality"].Value).Value; }
145      set { ((DoubleValue)Results["Best Quality"].Value).Value = value; }
146    }
147    private DataTable ResultsQualities
148    {
149      get { return ((DataTable)Results["Qualities"].Value); }
150    }
151    #endregion
152
153    [StorableConstructor]
154    protected GradientBoostedTreesAlgorithm(bool deserializing) : base(deserializing) { }
155
156    protected GradientBoostedTreesAlgorithm(GradientBoostedTreesAlgorithm original, Cloner cloner)
157      : base(original, cloner) {
158    }
159
160    public override IDeepCloneable Clone(Cloner cloner) {
161      return new GradientBoostedTreesAlgorithm(this, cloner);
162    }
163
164    public GradientBoostedTreesAlgorithm() {
165      Problem = new RegressionProblem(); // default problem
166
167      Parameters.Add(new FixedValueParameter<IntValue>(IterationsParameterName, "Number of iterations (set as high as possible, adjust in combination with nu, when increasing iterations also decrease nu)", new IntValue(1000)));
168      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
169      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
170      Parameters.Add(new FixedValueParameter<IntValue>(MaxSizeParameterName, "Maximal size of the tree learned in each step (prefer smaller sizes if possible)", new IntValue(10)));
171      Parameters.Add(new FixedValueParameter<DoubleValue>(RParameterName, "Ratio of training rows selected randomly in each step (0 < R <= 1)", new DoubleValue(0.5)));
172      Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName, "Ratio of variables selected randomly in each step (0 < M <= 1)", new DoubleValue(0.5)));
173      Parameters.Add(new FixedValueParameter<DoubleValue>(NuParameterName, "Learning rate nu (step size for the gradient update, should be small 0 < nu < 0.1)", new DoubleValue(0.002)));
174      Parameters.Add(new FixedValueParameter<IntValue>(UpdateIntervalParameterName, "", new IntValue(100)));
175      Parameters[UpdateIntervalParameterName].Hidden = true;
176      Parameters.Add(new FixedValueParameter<EnumValue<ModelStorage>>(CreateSolutionParameterName, "Flag that indicates what kind of solution should be produced at the end of the run", new EnumValue<ModelStorage>(ModelStorage.Parameter)));
177      Parameters[CreateSolutionParameterName].Hidden = true;
178
179      var lossFunctions = ApplicationManager.Manager.GetInstances<ILossFunction>();
180      Parameters.Add(new ConstrainedValueParameter<ILossFunction>(LossFunctionParameterName, "The loss function", new ItemSet<ILossFunction>(lossFunctions)));
181      LossFunctionParameter.Value = LossFunctionParameter.ValidValues.First(f => f.ToString().Contains("Squared")); // squared error loss is the default
182    }
183
184    [StorableHook(HookType.AfterDeserialization)]
185    private void AfterDeserialization() {
186      // BackwardsCompatibility3.4
187      #region Backwards compatible code, remove with 3.5
188
189      #region LossFunction
190      // parameter type has been changed
191      var lossFunctionParam = Parameters[LossFunctionParameterName] as ConstrainedValueParameter<StringValue>;
192      if (lossFunctionParam != null) {
193        Parameters.Remove(LossFunctionParameterName);
194        var selectedValue = lossFunctionParam.Value; // to be restored below
195
196        var lossFunctions = ApplicationManager.Manager.GetInstances<ILossFunction>();
197        Parameters.Add(new ConstrainedValueParameter<ILossFunction>(LossFunctionParameterName, "The loss function", new ItemSet<ILossFunction>(lossFunctions)));
198        // try to restore selected value
199        var selectedLossFunction =
200          LossFunctionParameter.ValidValues.FirstOrDefault(f => f.ToString() == selectedValue.Value);
201        if (selectedLossFunction != null) {
202          LossFunctionParameter.Value = selectedLossFunction;
203        } else {
204          LossFunctionParameter.Value = LossFunctionParameter.ValidValues.First(f => f.ToString().Contains("Squared")); // default: SE
205        }
206      }
207      #endregion
208
209      #region CreateSolution
210      // parameter type has been changed
211      var createSolutionParam = Parameters[CreateSolutionParameterName] as FixedValueParameter<BoolValue>;
212      if (createSolutionParam != null) {
213        Parameters.Remove(CreateSolutionParameterName);
214
215        ModelStorage value = createSolutionParam.Value.Value ? ModelStorage.Parameter : ModelStorage.Quality;
216        Parameters.Add(new FixedValueParameter<EnumValue<ModelStorage>>(CreateSolutionParameterName, "Flag that indicates what kind of solution should be produced at the end of the run", new EnumValue<ModelStorage>(value)));
217        Parameters[CreateSolutionParameterName].Hidden = true;
218      }
219      #endregion
220      #endregion
221    }
222
223    protected override void Run(CancellationToken cancellationToken) {
224      // Set up the algorithm
225      if (SetSeedRandomly) Seed = new System.Random().Next();
226
227      // Set up the results display
228      var iterations = new IntValue(0);
229      Results.Add(new Result("Iterations", iterations));
230
231      var table = new DataTable("Qualities");
232      table.Rows.Add(new DataRow("Loss (train)"));
233      table.Rows.Add(new DataRow("Loss (test)"));
234      table.Rows["Loss (train)"].VisualProperties.StartIndexZero = true;
235      table.Rows["Loss (test)"].VisualProperties.StartIndexZero = true;
236
237      Results.Add(new Result("Qualities", table));
238      var curLoss = new DoubleValue();
239      Results.Add(new Result("Loss (train)", curLoss));
240
241      // init
242      var problemData = (IRegressionProblemData)Problem.ProblemData.Clone();
243      var lossFunction = LossFunctionParameter.Value;
244      var state = GradientBoostedTreesAlgorithmStatic.CreateGbmState(problemData, lossFunction, (uint)Seed, MaxSize, R, M, Nu);
245
246      var updateInterval = UpdateIntervalParameter.Value.Value;
247      // Loop until iteration limit reached or canceled.
248      for (int i = 0; i < Iterations; i++) {
249        cancellationToken.ThrowIfCancellationRequested();
250
251        GradientBoostedTreesAlgorithmStatic.MakeStep(state);
252
253        // iteration results
254        if (i % updateInterval == 0) {
255          curLoss.Value = state.GetTrainLoss();
256          table.Rows["Loss (train)"].Values.Add(curLoss.Value);
257          table.Rows["Loss (test)"].Values.Add(state.GetTestLoss());
258          iterations.Value = i;
259        }
260      }
261
262      // final results
263      iterations.Value = Iterations;
264      curLoss.Value = state.GetTrainLoss();
265      table.Rows["Loss (train)"].Values.Add(curLoss.Value);
266      table.Rows["Loss (test)"].Values.Add(state.GetTestLoss());
267
268      // produce variable relevance
269      var orderedImpacts = state.GetVariableRelevance().Select(t => new { name = t.Key, impact = t.Value }).ToList();
270
271      var impacts = new DoubleMatrix();
272      var matrix = impacts as IStringConvertibleMatrix;
273      matrix.Rows = orderedImpacts.Count;
274      matrix.RowNames = orderedImpacts.Select(x => x.name);
275      matrix.Columns = 1;
276      matrix.ColumnNames = new string[] { "Relative variable relevance" };
277
278      int rowIdx = 0;
279      foreach (var p in orderedImpacts) {
280        matrix.SetValue(string.Format("{0:N2}", p.impact), rowIdx++, 0);
281      }
282
283      Results.Add(new Result("Variable relevance", impacts));
284      Results.Add(new Result("Loss (test)", new DoubleValue(state.GetTestLoss())));
285
286      // produce solution
287      if (CreateSolution == ModelStorage.Parameter || CreateSolution == ModelStorage.Complete) {
288        IRegressionModel model = state.GetModel();
289
290        if (CreateSolution == ModelStorage.Parameter) {
291          model = new GradientBoostedTreesModelSurrogate(problemData, (uint)Seed, lossFunction, Iterations, MaxSize, R, M, Nu, (GradientBoostedTreesModel)model);
292        }
293
294        // for logistic regression we produce a classification solution
295        if (lossFunction is LogisticRegressionLoss) {
296          var classificationModel = new DiscriminantFunctionClassificationModel(model,
297            new AccuracyMaximizationThresholdCalculator());
298          var classificationProblemData = new ClassificationProblemData(problemData.Dataset,
299            problemData.AllowedInputVariables, problemData.TargetVariable, problemData.Transformations);
300          classificationProblemData.TrainingPartition.Start = Problem.ProblemData.TrainingPartition.Start;
301          classificationProblemData.TrainingPartition.End = Problem.ProblemData.TrainingPartition.End;
302          classificationProblemData.TestPartition.Start = Problem.ProblemData.TestPartition.Start;
303          classificationProblemData.TestPartition.End = Problem.ProblemData.TestPartition.End;
304
305          classificationModel.SetThresholdsAndClassValues(new double[] { double.NegativeInfinity, 0.0 }, new[] { 0.0, 1.0 });
306
307
308          var classificationSolution = new DiscriminantFunctionClassificationSolution(classificationModel, classificationProblemData);
309          Results.Add(new Result("Solution", classificationSolution));
310        } else {
311          // otherwise we produce a regression solution
312          Results.Add(new Result("Solution", new GradientBoostedTreesSolution(model, problemData)));
313        }
314      } else if (CreateSolution == ModelStorage.Quality) {
315        //Do nothing
316      } else {
317        throw new NotImplementedException("Selected parameter for CreateSolution isn't implemented yet");
318      }
319    }
320  }
321}
Note: See TracBrowser for help on using the repository browser.