source: branches/GBT/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithm.cs @ 12332

Last change on this file since 12332 was 12332, checked in by gkronber, 5 years ago

#2261: initial import of gradient boosted trees for regression

File size: 10.3 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 * and the BEACON Center for the Study of Evolution in Action.
5 *
6 * This file is part of HeuristicLab.
7 *
8 * HeuristicLab is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * (at your option) any later version.
12 *
13 * HeuristicLab is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
20 */
21#endregion
22
23using System;
24using System.Collections.Generic;
25using System.ComponentModel;
26using System.Diagnostics.Contracts;
27using System.Linq;
28using System.Threading;
29using GradientBoostedTrees;
30using HeuristicLab.Analysis;
31using HeuristicLab.Common;
32using HeuristicLab.Core;
33using HeuristicLab.Data;
34using HeuristicLab.Optimization;
35using HeuristicLab.Parameters;
36using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
37using HeuristicLab.PluginInfrastructure;
38using HeuristicLab.Problems.DataAnalysis;
39using HeuristicLab.Random;
40
41namespace HeuristicLab.Algorithms.DataAnalysis {
42  [Item("Gradient Boosted Trees", "")]
43  [StorableClass]
44  [Creatable("Algorithms")]
45  public class GradientBoostedTreesAlgorithm : BasicAlgorithm {
46    public override Type ProblemType {
47      get { return typeof(IRegressionProblem); }
48    }
49    public new IRegressionProblem Problem {
50      get { return (IRegressionProblem)base.Problem; }
51      set { base.Problem = value; }
52    }
53
54    #region ParameterNames
55    private const string IterationsParameterName = "Iterations";
56    private const string MaxDepthParameterName = "Maximum Tree Depth";
57    private const string NuParameterName = "Nu";
58    private const string RParameterName = "R";
59    private const string MParameterName = "M";
60    private const string SeedParameterName = "Seed";
61    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
62    private const string LossFunctionParameterName = "LossFunction";
63    private const string UpdateIntervalParameterName = "UpdateInterval";
64    #endregion
65
66    #region ParameterProperties
67    public IFixedValueParameter<IntValue> IterationsParameter {
68      get { return (IFixedValueParameter<IntValue>)Parameters[IterationsParameterName]; }
69    }
70    public IFixedValueParameter<IntValue> MaxDepthParameter {
71      get { return (IFixedValueParameter<IntValue>)Parameters[MaxDepthParameterName]; }
72    }
73    public IFixedValueParameter<DoubleValue> NuParameter {
74      get { return (IFixedValueParameter<DoubleValue>)Parameters[NuParameterName]; }
75    }
76    public IFixedValueParameter<DoubleValue> RParameter {
77      get { return (IFixedValueParameter<DoubleValue>)Parameters[RParameterName]; }
78    }
79    public IFixedValueParameter<DoubleValue> MParameter {
80      get { return (IFixedValueParameter<DoubleValue>)Parameters[MParameterName]; }
81    }
82    public IFixedValueParameter<IntValue> SeedParameter {
83      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
84    }
85    public FixedValueParameter<BoolValue> SetSeedRandomlyParameter {
86      get { return (FixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
87    }
88    public IConstrainedValueParameter<StringValue> LossFunctionParameter {
89      get { return (IConstrainedValueParameter<StringValue>)Parameters[LossFunctionParameterName]; }
90    }
91    public IFixedValueParameter<IntValue> UpdateIntervalParameter {
92      get { return (IFixedValueParameter<IntValue>)Parameters[UpdateIntervalParameterName]; }
93    }
94    #endregion
95
96    #region Properties
97    public int Iterations {
98      get { return IterationsParameter.Value.Value; }
99      set { IterationsParameter.Value.Value = value; }
100    }
101    public int Seed {
102      get { return SeedParameter.Value.Value; }
103      set { SeedParameter.Value.Value = value; }
104    }
105    public bool SetSeedRandomly {
106      get { return SetSeedRandomlyParameter.Value.Value; }
107      set { SetSeedRandomlyParameter.Value.Value = value; }
108    }
109    public int MaxDepth {
110      get { return MaxDepthParameter.Value.Value; }
111      set { MaxDepthParameter.Value.Value = value; }
112    }
113    public double Nu {
114      get { return NuParameter.Value.Value; }
115      set { NuParameter.Value.Value = value; }
116    }
117    public double R {
118      get { return RParameter.Value.Value; }
119      set { RParameter.Value.Value = value; }
120    }
121    public double M {
122      get { return MParameter.Value.Value; }
123      set { MParameter.Value.Value = value; }
124    }
125    #endregion
126
127    #region ResultsProperties
128    private double ResultsBestQuality {
129      get { return ((DoubleValue)Results["Best Quality"].Value).Value; }
130      set { ((DoubleValue)Results["Best Quality"].Value).Value = value; }
131    }
132    private DataTable ResultsQualities {
133      get { return ((DataTable)Results["Qualities"].Value); }
134    }
135    #endregion
136
137    [StorableConstructor]
138    protected GradientBoostedTreesAlgorithm(bool deserializing) : base(deserializing) { }
139
140    protected GradientBoostedTreesAlgorithm(GradientBoostedTreesAlgorithm original, Cloner cloner)
141      : base(original, cloner) {
142    }
143
144    public override IDeepCloneable Clone(Cloner cloner) {
145      return new GradientBoostedTreesAlgorithm(this, cloner);
146    }
147
148    public GradientBoostedTreesAlgorithm() {
149      Problem = new RegressionProblem(); // default problem
150
151      Parameters.Add(new FixedValueParameter<IntValue>(IterationsParameterName, "Number of iterations (set as high as possible, adjust in combination with nu, when increasing iterations also decrease nu)", new IntValue(1000)));
152      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
153      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
154      Parameters.Add(new FixedValueParameter<IntValue>(MaxDepthParameterName, "Maximal depth of the tree learned in each step (prefer smaller depths if possible)", new IntValue(5)));
155      Parameters.Add(new FixedValueParameter<DoubleValue>(RParameterName, "Ratio of training rows selected randomly in each step (0 < R <= 1)", new DoubleValue(0.5)));
156      Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName, "Ratio of variables selected randomly in each step (0 < M <= 1)", new DoubleValue(0.5)));
157      Parameters.Add(new FixedValueParameter<DoubleValue>(NuParameterName, "Learning rate nu (step size for the gradient update, should be small 0 < nu < 0.1)", new DoubleValue(0.002)));
158      Parameters.Add(new FixedValueParameter<IntValue>("UpdateInterval", "", new IntValue(100)));
159      Parameters["UpdateInterval"].Hidden = true;
160
161      var lossFunctionNames = ApplicationManager.Manager.GetInstances<ILossFunction>().Select(l => new StringValue(l.ToString()).AsReadOnly());
162      var defaultLossFunction = lossFunctionNames.First(l => l.Value.Contains("Squared")); // squared error loss is the default
163      Parameters.Add(new ConstrainedValueParameter<StringValue>(LossFunctionParameterName, "The loss function", new ItemSet<StringValue>(lossFunctionNames), defaultLossFunction));
164    }
165
166
167    protected override void Run(CancellationToken cancellationToken) {
168      // Set up the algorithm
169      if (SetSeedRandomly) Seed = new System.Random().Next();
170      // random.Reset(Seed);
171
172      // Set up the results display
173      var iterations = new IntValue(0);
174      Results.Add(new Result("Iterations", iterations));
175
176      var table = new DataTable("Qualities");
177      table.Rows.Add(new DataRow("Loss (train)"));
178      table.Rows.Add(new DataRow("Loss (test)"));
179      Results.Add(new Result("Qualities", table));
180      var curLoss = new DoubleValue();
181      Results.Add(new Result("Current loss", curLoss));
182
183      // init
184      var problemData = Problem.ProblemData;
185      var lossFunction = ApplicationManager.Manager.GetInstances<ILossFunction>().Single(l => l.ToString() == LossFunctionParameter.Value.Value);
186      var state = GradientBoostedTreesAlgorithmStatic.CreateGbmState(problemData, lossFunction, (uint)Seed, MaxDepth, R, M, Nu);
187
188      var updateInterval = UpdateIntervalParameter.Value.Value;
189      // Loop until iteration limit reached or canceled.
190      for (int i = 0; i < Iterations; i++) {
191        cancellationToken.ThrowIfCancellationRequested();
192
193        GradientBoostedTreesAlgorithmStatic.MakeStep(state);
194
195        // iteration results
196        if (i % updateInterval == 0) {
197          curLoss.Value = state.GetTrainLoss();
198          table.Rows["Loss (train)"].Values.Add(curLoss.Value);
199          table.Rows["Loss (test)"].Values.Add(state.GetTestLoss());
200          iterations.Value = i;
201        }
202      }
203
204      // final results
205      iterations.Value = Iterations;
206      curLoss.Value = state.GetTrainLoss();
207      table.Rows["Loss (train)"].Values.Add(curLoss.Value);
208      table.Rows["Loss (test)"].Values.Add(state.GetTestLoss());
209
210      // produce variable relevance
211      // update variable impacts matrix
212      var orderedImpacts = state.GetVariableRelevance().Select(t => new { name = t.Key, impact = t.Value }).ToList();
213
214      var impacts = new DoubleMatrix();
215      var matrix = impacts as IStringConvertibleMatrix;
216      matrix.Rows = orderedImpacts.Count;
217      matrix.RowNames = orderedImpacts.Select(x => x.name);
218      matrix.Columns = 1;
219      matrix.ColumnNames = new string[] { "Relative variable relevance" };
220
221      int rowIdx = 0;
222      foreach (var p in orderedImpacts) {
223        matrix.SetValue(string.Format("{0:N2}", p.impact), rowIdx++, 0);
224      }
225
226      Results.Add(new Result("Variable relevance", impacts));
227
228      // produce solution
229      Results.Add(new Result("Solution", new RegressionSolution(state.GetModel(), (IRegressionProblemData)problemData.Clone())));
230    }
231  }
232}
Note: See TracBrowser for help on using the repository browser.