source: branches/TSNE/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithm.cs @ 14836

Last change on this file since 14836 was 14836, checked in by gkronber, 2 years ago

#2700 merged changesets from trunk to branch

File size: 13.4 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 * and the BEACON Center for the Study of Evolution in Action.
5 *
6 * This file is part of HeuristicLab.
7 *
8 * HeuristicLab is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * (at your option) any later version.
12 *
13 * HeuristicLab is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
20 */
21#endregion
22
23using System;
24using System.Linq;
25using System.Threading;
26using HeuristicLab.Analysis;
27using HeuristicLab.Common;
28using HeuristicLab.Core;
29using HeuristicLab.Data;
30using HeuristicLab.Optimization;
31using HeuristicLab.Parameters;
32using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
33using HeuristicLab.PluginInfrastructure;
34using HeuristicLab.Problems.DataAnalysis;
35
36namespace HeuristicLab.Algorithms.DataAnalysis {
37  [Item("Gradient Boosted Trees (GBT)", "Gradient boosted trees algorithm. Specific implementation of gradient boosting for regression trees. Friedman, J. \"Greedy Function Approximation: A Gradient Boosting Machine\", IMS 1999 Reitz Lecture.")]
38  [StorableClass]
39  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 125)]
40  public class GradientBoostedTreesAlgorithm : FixedDataAnalysisAlgorithm<IRegressionProblem> {
41    #region ParameterNames
42    private const string IterationsParameterName = "Iterations";
43    private const string MaxSizeParameterName = "Maximum Tree Size";
44    private const string NuParameterName = "Nu";
45    private const string RParameterName = "R";
46    private const string MParameterName = "M";
47    private const string SeedParameterName = "Seed";
48    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
49    private const string LossFunctionParameterName = "LossFunction";
50    private const string UpdateIntervalParameterName = "UpdateInterval";
51    private const string CreateSolutionParameterName = "CreateSolution";
52    #endregion
53
54    #region ParameterProperties
55    public IFixedValueParameter<IntValue> IterationsParameter
56    {
57      get { return (IFixedValueParameter<IntValue>)Parameters[IterationsParameterName]; }
58    }
59    public IFixedValueParameter<IntValue> MaxSizeParameter
60    {
61      get { return (IFixedValueParameter<IntValue>)Parameters[MaxSizeParameterName]; }
62    }
63    public IFixedValueParameter<DoubleValue> NuParameter
64    {
65      get { return (IFixedValueParameter<DoubleValue>)Parameters[NuParameterName]; }
66    }
67    public IFixedValueParameter<DoubleValue> RParameter
68    {
69      get { return (IFixedValueParameter<DoubleValue>)Parameters[RParameterName]; }
70    }
71    public IFixedValueParameter<DoubleValue> MParameter
72    {
73      get { return (IFixedValueParameter<DoubleValue>)Parameters[MParameterName]; }
74    }
75    public IFixedValueParameter<IntValue> SeedParameter
76    {
77      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
78    }
79    public FixedValueParameter<BoolValue> SetSeedRandomlyParameter
80    {
81      get { return (FixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
82    }
83    public IConstrainedValueParameter<ILossFunction> LossFunctionParameter
84    {
85      get { return (IConstrainedValueParameter<ILossFunction>)Parameters[LossFunctionParameterName]; }
86    }
87    public IFixedValueParameter<IntValue> UpdateIntervalParameter
88    {
89      get { return (IFixedValueParameter<IntValue>)Parameters[UpdateIntervalParameterName]; }
90    }
91    public IFixedValueParameter<BoolValue> CreateSolutionParameter
92    {
93      get { return (IFixedValueParameter<BoolValue>)Parameters[CreateSolutionParameterName]; }
94    }
95    #endregion
96
97    #region Properties
98    public int Iterations
99    {
100      get { return IterationsParameter.Value.Value; }
101      set { IterationsParameter.Value.Value = value; }
102    }
103    public int Seed
104    {
105      get { return SeedParameter.Value.Value; }
106      set { SeedParameter.Value.Value = value; }
107    }
108    public bool SetSeedRandomly
109    {
110      get { return SetSeedRandomlyParameter.Value.Value; }
111      set { SetSeedRandomlyParameter.Value.Value = value; }
112    }
113    public int MaxSize
114    {
115      get { return MaxSizeParameter.Value.Value; }
116      set { MaxSizeParameter.Value.Value = value; }
117    }
118    public double Nu
119    {
120      get { return NuParameter.Value.Value; }
121      set { NuParameter.Value.Value = value; }
122    }
123    public double R
124    {
125      get { return RParameter.Value.Value; }
126      set { RParameter.Value.Value = value; }
127    }
128    public double M
129    {
130      get { return MParameter.Value.Value; }
131      set { MParameter.Value.Value = value; }
132    }
133    public bool CreateSolution
134    {
135      get { return CreateSolutionParameter.Value.Value; }
136      set { CreateSolutionParameter.Value.Value = value; }
137    }
138    #endregion
139
140    #region ResultsProperties
141    private double ResultsBestQuality
142    {
143      get { return ((DoubleValue)Results["Best Quality"].Value).Value; }
144      set { ((DoubleValue)Results["Best Quality"].Value).Value = value; }
145    }
146    private DataTable ResultsQualities
147    {
148      get { return ((DataTable)Results["Qualities"].Value); }
149    }
150    #endregion
151
152    [StorableConstructor]
153    protected GradientBoostedTreesAlgorithm(bool deserializing) : base(deserializing) { }
154
155    protected GradientBoostedTreesAlgorithm(GradientBoostedTreesAlgorithm original, Cloner cloner)
156      : base(original, cloner) {
157    }
158
159    public override IDeepCloneable Clone(Cloner cloner) {
160      return new GradientBoostedTreesAlgorithm(this, cloner);
161    }
162
163    public GradientBoostedTreesAlgorithm() {
164      Problem = new RegressionProblem(); // default problem
165
166      Parameters.Add(new FixedValueParameter<IntValue>(IterationsParameterName, "Number of iterations (set as high as possible, adjust in combination with nu, when increasing iterations also decrease nu)", new IntValue(1000)));
167      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
168      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
169      Parameters.Add(new FixedValueParameter<IntValue>(MaxSizeParameterName, "Maximal size of the tree learned in each step (prefer smaller sizes if possible)", new IntValue(10)));
170      Parameters.Add(new FixedValueParameter<DoubleValue>(RParameterName, "Ratio of training rows selected randomly in each step (0 < R <= 1)", new DoubleValue(0.5)));
171      Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName, "Ratio of variables selected randomly in each step (0 < M <= 1)", new DoubleValue(0.5)));
172      Parameters.Add(new FixedValueParameter<DoubleValue>(NuParameterName, "Learning rate nu (step size for the gradient update, should be small 0 < nu < 0.1)", new DoubleValue(0.002)));
173      Parameters.Add(new FixedValueParameter<IntValue>(UpdateIntervalParameterName, "", new IntValue(100)));
174      Parameters[UpdateIntervalParameterName].Hidden = true;
175      Parameters.Add(new FixedValueParameter<BoolValue>(CreateSolutionParameterName, "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
176      Parameters[CreateSolutionParameterName].Hidden = true;
177
178      var lossFunctions = ApplicationManager.Manager.GetInstances<ILossFunction>();
179      Parameters.Add(new ConstrainedValueParameter<ILossFunction>(LossFunctionParameterName, "The loss function", new ItemSet<ILossFunction>(lossFunctions)));
180      LossFunctionParameter.Value = LossFunctionParameter.ValidValues.First(f => f.ToString().Contains("Squared")); // squared error loss is the default
181    }
182
183    [StorableHook(HookType.AfterDeserialization)]
184    private void AfterDeserialization() {
185      // BackwardsCompatibility3.4
186      #region Backwards compatible code, remove with 3.5
187      // parameter type has been changed
188      var lossFunctionParam = Parameters[LossFunctionParameterName] as ConstrainedValueParameter<StringValue>;
189      if (lossFunctionParam != null) {
190        Parameters.Remove(LossFunctionParameterName);
191        var selectedValue = lossFunctionParam.Value; // to be restored below
192
193        var lossFunctions = ApplicationManager.Manager.GetInstances<ILossFunction>();
194        Parameters.Add(new ConstrainedValueParameter<ILossFunction>(LossFunctionParameterName, "The loss function", new ItemSet<ILossFunction>(lossFunctions)));
195        // try to restore selected value
196        var selectedLossFunction =
197          LossFunctionParameter.ValidValues.FirstOrDefault(f => f.ToString() == selectedValue.Value);
198        if (selectedLossFunction != null) {
199          LossFunctionParameter.Value = selectedLossFunction;
200        } else {
201          LossFunctionParameter.Value = LossFunctionParameter.ValidValues.First(f => f.ToString().Contains("Squared")); // default: SE
202        }
203      }
204      #endregion
205    }
206
207    protected override void Run(CancellationToken cancellationToken) {
208      // Set up the algorithm
209      if (SetSeedRandomly) Seed = new System.Random().Next();
210
211      // Set up the results display
212      var iterations = new IntValue(0);
213      Results.Add(new Result("Iterations", iterations));
214
215      var table = new DataTable("Qualities");
216      table.Rows.Add(new DataRow("Loss (train)"));
217      table.Rows.Add(new DataRow("Loss (test)"));
218      Results.Add(new Result("Qualities", table));
219      var curLoss = new DoubleValue();
220      Results.Add(new Result("Loss (train)", curLoss));
221
222      // init
223      var problemData = (IRegressionProblemData)Problem.ProblemData.Clone();
224      var lossFunction = LossFunctionParameter.Value;
225      var state = GradientBoostedTreesAlgorithmStatic.CreateGbmState(problemData, lossFunction, (uint)Seed, MaxSize, R, M, Nu);
226
227      var updateInterval = UpdateIntervalParameter.Value.Value;
228      // Loop until iteration limit reached or canceled.
229      for (int i = 0; i < Iterations; i++) {
230        cancellationToken.ThrowIfCancellationRequested();
231
232        GradientBoostedTreesAlgorithmStatic.MakeStep(state);
233
234        // iteration results
235        if (i % updateInterval == 0) {
236          curLoss.Value = state.GetTrainLoss();
237          table.Rows["Loss (train)"].Values.Add(curLoss.Value);
238          table.Rows["Loss (test)"].Values.Add(state.GetTestLoss());
239          iterations.Value = i;
240        }
241      }
242
243      // final results
244      iterations.Value = Iterations;
245      curLoss.Value = state.GetTrainLoss();
246      table.Rows["Loss (train)"].Values.Add(curLoss.Value);
247      table.Rows["Loss (test)"].Values.Add(state.GetTestLoss());
248
249      // produce variable relevance
250      var orderedImpacts = state.GetVariableRelevance().Select(t => new { name = t.Key, impact = t.Value }).ToList();
251
252      var impacts = new DoubleMatrix();
253      var matrix = impacts as IStringConvertibleMatrix;
254      matrix.Rows = orderedImpacts.Count;
255      matrix.RowNames = orderedImpacts.Select(x => x.name);
256      matrix.Columns = 1;
257      matrix.ColumnNames = new string[] { "Relative variable relevance" };
258
259      int rowIdx = 0;
260      foreach (var p in orderedImpacts) {
261        matrix.SetValue(string.Format("{0:N2}", p.impact), rowIdx++, 0);
262      }
263
264      Results.Add(new Result("Variable relevance", impacts));
265      Results.Add(new Result("Loss (test)", new DoubleValue(state.GetTestLoss())));
266
267      // produce solution
268      if (CreateSolution) {
269        var model = state.GetModel();
270
271        // for logistic regression we produce a classification solution
272        if (lossFunction is LogisticRegressionLoss) {
273          var classificationModel = new DiscriminantFunctionClassificationModel(model,
274            new AccuracyMaximizationThresholdCalculator());
275          var classificationProblemData = new ClassificationProblemData(problemData.Dataset,
276            problemData.AllowedInputVariables, problemData.TargetVariable, problemData.Transformations);
277          classificationProblemData.TrainingPartition.Start = Problem.ProblemData.TrainingPartition.Start;
278          classificationProblemData.TrainingPartition.End = Problem.ProblemData.TrainingPartition.End;
279          classificationProblemData.TestPartition.Start = Problem.ProblemData.TestPartition.Start;
280          classificationProblemData.TestPartition.End = Problem.ProblemData.TestPartition.End;
281
282          classificationModel.SetThresholdsAndClassValues(new double[] { double.NegativeInfinity, 0.0 }, new []{ 0.0, 1.0 });
283
284
285          var classificationSolution = new DiscriminantFunctionClassificationSolution(classificationModel, classificationProblemData);
286          Results.Add(new Result("Solution", classificationSolution));
287        } else {
288          // otherwise we produce a regression solution
289          Results.Add(new Result("Solution", new GradientBoostedTreesSolution(model, problemData)));
290        }
291      }
292    }
293  }
294}
Note: See TracBrowser for help on using the repository browser.