Free cookie consent management tool by TermsFeed Policy Generator

source: branches/DataAnalysis SolutionEnsembles/HeuristicLab.Problems.DataAnalysis.Classification/3.3/Symbolic/Analyzer/ValidationBestSymbolicClassificationSolutionAnalyzer.cs @ 5991

Last change on this file since 5991 was 5445, checked in by swagner, 14 years ago

Updated year of copyrights (#1406)
File size: 22.4 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2011 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System.Collections.Generic;
23using System.Linq;
24using HeuristicLab.Analysis;
25using HeuristicLab.Common;
26using HeuristicLab.Core;
27using HeuristicLab.Data;
28using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
29using HeuristicLab.Operators;
30using HeuristicLab.Optimization;
31using HeuristicLab.Parameters;
32using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
33using HeuristicLab.Problems.DataAnalysis.Regression.Symbolic;
34using HeuristicLab.Problems.DataAnalysis.Regression.Symbolic.Analyzers;
35using HeuristicLab.Problems.DataAnalysis.Symbolic;
36
37namespace HeuristicLab.Problems.DataAnalysis.Classification {
38  [Item("ValidationBestSymbolicClassificationSolutionAnalyzer", "An operator that analyzes the validation best symbolic classification solution.")]
39  [StorableClass]
40  public class ValidationBestSymbolicClassificationSolutionAnalyzer : SingleSuccessorOperator, ISymbolicClassificationAnalyzer {
41    private const string MaximizationParameterName = "Maximization";
42    private const string GenerationsParameterName = "Generations";
43    private const string RandomParameterName = "Random";
44    private const string SymbolicExpressionTreeParameterName = "SymbolicExpressionTree";
45    private const string SymbolicExpressionTreeInterpreterParameterName = "SymbolicExpressionTreeInterpreter";
46
47    private const string ClassificationProblemDataParameterName = "ClassificationProblemData";
48    private const string EvaluatorParameterName = "Evaluator";
49    private const string ValidationSamplesStartParameterName = "SamplesStart";
50    private const string ValidationSamplesEndParameterName = "SamplesEnd";
51    private const string RelativeNumberOfEvaluatedSamplesParameterName = "RelativeNumberOfEvaluatedSamples";
52    private const string UpperEstimationLimitParameterName = "UpperEstimationLimit";
53    private const string LowerEstimationLimitParameterName = "LowerEstimationLimit";
54    private const string CalculateSolutionComplexityParameterName = "CalculateSolutionComplexity";
55    private const string ApplyLinearScalingParameterName = "ApplyLinearScaling";
56
57    private const string ResultsParameterName = "Results";
58    private const string BestValidationQualityParameterName = "Best validation quality";
59    private const string BestValidationSolutionParameterName = "Best validation solution";
60    private const string BestSolutionAccuracyTrainingParameterName = "Best solution accuracy (training)";
61    private const string BestSolutionAccuracyTestParameterName = "Best solution accuracy (test)";
62    private const string BestSolutionLengthParameterName = "Best solution length (on validation set)";
63    private const string BestSolutionHeightParameterName = "Best solution height (on validation set)";
64    private const string VariableFrequenciesParameterName = "VariableFrequencies";
65
66    #region parameter properties
67    public ILookupParameter<BoolValue> MaximizationParameter {
68      get { return (ILookupParameter<BoolValue>)Parameters[MaximizationParameterName]; }
69    }
70    public ILookupParameter<IntValue> GenerationsParameter {
71      get { return (ILookupParameter<IntValue>)Parameters[GenerationsParameterName]; }
72    }
73    public ILookupParameter<IRandom> RandomParameter {
74      get { return (ILookupParameter<IRandom>)Parameters[RandomParameterName]; }
75    }
76    public ScopeTreeLookupParameter<SymbolicExpressionTree> SymbolicExpressionTreeParameter {
77      get { return (ScopeTreeLookupParameter<SymbolicExpressionTree>)Parameters[SymbolicExpressionTreeParameterName]; }
78    }
79    public IValueLookupParameter<ISymbolicExpressionTreeInterpreter> SymbolicExpressionTreeInterpreterParameter {
80      get { return (IValueLookupParameter<ISymbolicExpressionTreeInterpreter>)Parameters[SymbolicExpressionTreeInterpreterParameterName]; }
81    }
82    public ILookupParameter<ClassificationProblemData> ClassificationProblemDataParameter {
83      get { return (ILookupParameter<ClassificationProblemData>)Parameters[ClassificationProblemDataParameterName]; }
84    }
85    public ILookupParameter<ISymbolicClassificationEvaluator> EvaluatorParameter {
86      get { return (ILookupParameter<ISymbolicClassificationEvaluator>)Parameters[EvaluatorParameterName]; }
87    }
88    public IValueLookupParameter<IntValue> ValidationSamplesStartParameter {
89      get { return (IValueLookupParameter<IntValue>)Parameters[ValidationSamplesStartParameterName]; }
90    }
91    public IValueLookupParameter<IntValue> ValidationSamplesEndParameter {
92      get { return (IValueLookupParameter<IntValue>)Parameters[ValidationSamplesEndParameterName]; }
93    }
94    public IValueParameter<PercentValue> RelativeNumberOfEvaluatedSamplesParameter {
95      get { return (IValueParameter<PercentValue>)Parameters[RelativeNumberOfEvaluatedSamplesParameterName]; }
96    }
97    public IValueLookupParameter<DoubleValue> UpperEstimationLimitParameter {
98      get { return (IValueLookupParameter<DoubleValue>)Parameters[UpperEstimationLimitParameterName]; }
99    }
100    public IValueLookupParameter<DoubleValue> LowerEstimationLimitParameter {
101      get { return (IValueLookupParameter<DoubleValue>)Parameters[LowerEstimationLimitParameterName]; }
102    }
103    public IValueLookupParameter<BoolValue> ApplyLinearScalingParameter {
104      get { return (IValueLookupParameter<BoolValue>)Parameters[ApplyLinearScalingParameterName]; }
105    }
106    public ILookupParameter<DataTable> VariableFrequenciesParameter {
107      get { return (ILookupParameter<DataTable>)Parameters[VariableFrequenciesParameterName]; }
108    }
109    public IValueParameter<BoolValue> CalculateSolutionComplexityParameter {
110      get { return (IValueParameter<BoolValue>)Parameters[CalculateSolutionComplexityParameterName]; }
111    }
112
113    public ILookupParameter<ResultCollection> ResultsParameter {
114      get { return (ILookupParameter<ResultCollection>)Parameters[ResultsParameterName]; }
115    }
116    public ILookupParameter<DoubleValue> BestValidationQualityParameter {
117      get { return (ILookupParameter<DoubleValue>)Parameters[BestValidationQualityParameterName]; }
118    }
119    public ILookupParameter<SymbolicClassificationSolution> BestValidationSolutionParameter {
120      get { return (ILookupParameter<SymbolicClassificationSolution>)Parameters[BestValidationSolutionParameterName]; }
121    }
122    public ILookupParameter<DoubleValue> BestSolutionAccuracyTrainingParameter {
123      get { return (ILookupParameter<DoubleValue>)Parameters[BestSolutionAccuracyTrainingParameterName]; }
124    }
125    public ILookupParameter<DoubleValue> BestSolutionAccuracyTestParameter {
126      get { return (ILookupParameter<DoubleValue>)Parameters[BestSolutionAccuracyTestParameterName]; }
127    }
128    public ILookupParameter<IntValue> BestSolutionLengthParameter {
129      get { return (ILookupParameter<IntValue>)Parameters[BestSolutionLengthParameterName]; }
130    }
131    public ILookupParameter<IntValue> BestSolutionHeightParameter {
132      get { return (ILookupParameter<IntValue>)Parameters[BestSolutionHeightParameterName]; }
133    }
134    #endregion
135    #region properties
136    public BoolValue Maximization {
137      get { return MaximizationParameter.ActualValue; }
138    }
139    public IntValue Generations {
140      get { return GenerationsParameter.ActualValue; }
141    }
142    public IRandom Random {
143      get { return RandomParameter.ActualValue; }
144    }
145    public ItemArray<SymbolicExpressionTree> SymbolicExpressionTree {
146      get { return SymbolicExpressionTreeParameter.ActualValue; }
147    }
148    public ISymbolicExpressionTreeInterpreter SymbolicExpressionTreeInterpreter {
149      get { return SymbolicExpressionTreeInterpreterParameter.ActualValue; }
150    }
151
152    public ClassificationProblemData ClassificationProblemData {
153      get { return ClassificationProblemDataParameter.ActualValue; }
154    }
155    public ISymbolicClassificationEvaluator Evaluator {
156      get { return EvaluatorParameter.ActualValue; }
157    }
158    public IntValue ValidiationSamplesStart {
159      get { return ValidationSamplesStartParameter.ActualValue; }
160    }
161    public IntValue ValidationSamplesEnd {
162      get { return ValidationSamplesEndParameter.ActualValue; }
163    }
164    public PercentValue RelativeNumberOfEvaluatedSamples {
165      get { return RelativeNumberOfEvaluatedSamplesParameter.Value; }
166    }
167    public DoubleValue UpperEstimationLimit {
168      get { return UpperEstimationLimitParameter.ActualValue; }
169    }
170    public DoubleValue LowerEstimationLimit {
171      get { return LowerEstimationLimitParameter.ActualValue; }
172    }
173    public BoolValue ApplyLinearScaling {
174      get { return ApplyLinearScalingParameter.ActualValue; }
175      set { ApplyLinearScalingParameter.ActualValue = value; }
176    }
177    public DataTable VariableFrequencies {
178      get { return VariableFrequenciesParameter.ActualValue; }
179    }
180    public BoolValue CalculateSolutionComplexity {
181      get { return CalculateSolutionComplexityParameter.Value; }
182      set { CalculateSolutionComplexityParameter.Value = value; }
183    }
184
185    public ResultCollection Results {
186      get { return ResultsParameter.ActualValue; }
187    }
188    public DoubleValue BestValidationQuality {
189      get { return BestValidationQualityParameter.ActualValue; }
190      protected set { BestValidationQualityParameter.ActualValue = value; }
191    }
192    public SymbolicClassificationSolution BestValidationSolution {
193      get { return BestValidationSolutionParameter.ActualValue; }
194      protected set { BestValidationSolutionParameter.ActualValue = value; }
195    }
196    public DoubleValue BestSolutionAccuracyTraining {
197      get { return BestSolutionAccuracyTrainingParameter.ActualValue; }
198      protected set { BestSolutionAccuracyTrainingParameter.ActualValue = value; }
199    }
200    public DoubleValue BestSolutionAccuracyTest {
201      get { return BestSolutionAccuracyTestParameter.ActualValue; }
202      protected set { BestSolutionAccuracyTestParameter.ActualValue = value; }
203    }
204    public IntValue BestSolutionLength {
205      get { return BestSolutionLengthParameter.ActualValue; }
206      set { BestSolutionLengthParameter.ActualValue = value; }
207    }
208    public IntValue BestSolutionHeight {
209      get { return BestSolutionHeightParameter.ActualValue; }
210      set { BestSolutionHeightParameter.ActualValue = value; }
211    }
212    #endregion
213
214    [StorableConstructor]
215    protected ValidationBestSymbolicClassificationSolutionAnalyzer(bool deserializing) : base(deserializing) { }
216    protected ValidationBestSymbolicClassificationSolutionAnalyzer(ValidationBestSymbolicClassificationSolutionAnalyzer original, Cloner cloner)
217      : base(original, cloner) {
218    }
219    public ValidationBestSymbolicClassificationSolutionAnalyzer()
220      : base() {
221      Parameters.Add(new LookupParameter<BoolValue>(MaximizationParameterName, "The direction of optimization."));
222      Parameters.Add(new LookupParameter<IntValue>(GenerationsParameterName, "The number of generations calculated so far."));
223      Parameters.Add(new LookupParameter<IRandom>(RandomParameterName, "The random generator to use."));
224      Parameters.Add(new ScopeTreeLookupParameter<SymbolicExpressionTree>(SymbolicExpressionTreeParameterName, "The symbolic expression trees to analyze."));
225      Parameters.Add(new ValueLookupParameter<ISymbolicExpressionTreeInterpreter>(SymbolicExpressionTreeInterpreterParameterName, "The interpreter that should be used for the analysis of symbolic expression trees."));
226      Parameters.Add(new LookupParameter<ClassificationProblemData>(ClassificationProblemDataParameterName, "The problem data for which the symbolic expression tree is a solution."));
227      Parameters.Add(new LookupParameter<ISymbolicClassificationEvaluator>(EvaluatorParameterName, "The evaluator which should be used to evaluate the solution on the validation set."));
228      Parameters.Add(new ValueLookupParameter<IntValue>(ValidationSamplesStartParameterName, "The first index of the validation partition of the data set."));
229      Parameters.Add(new ValueLookupParameter<IntValue>(ValidationSamplesEndParameterName, "The last index of the validation partition of the data set."));
230      Parameters.Add(new ValueParameter<PercentValue>(RelativeNumberOfEvaluatedSamplesParameterName, "The relative number of samples of the dataset partition, which should be randomly chosen for evaluation between the start and end index.", new PercentValue(1)));
231      Parameters.Add(new ValueLookupParameter<DoubleValue>(UpperEstimationLimitParameterName, "The upper estimation limit that was set for the evaluation of the symbolic expression trees."));
232      Parameters.Add(new ValueLookupParameter<DoubleValue>(LowerEstimationLimitParameterName, "The lower estimation limit that was set for the evaluation of the symbolic expression trees."));
233      Parameters.Add(new LookupParameter<DataTable>(VariableFrequenciesParameterName, "The variable frequencies table to use for the calculation of variable impacts"));
234      Parameters.Add(new ValueParameter<BoolValue>(CalculateSolutionComplexityParameterName, "Determines if the length and height of the validation best solution should be calculated.", new BoolValue(true)));
235      Parameters.Add(new ValueLookupParameter<BoolValue>(ApplyLinearScalingParameterName, "The switch determines if the best solution should be linearly scaled on the whole training set.", new BoolValue(false)));
236
237      Parameters.Add(new ValueLookupParameter<ResultCollection>(ResultsParameterName, "The results collection where the analysis values should be stored."));
238      Parameters.Add(new LookupParameter<DoubleValue>(BestValidationQualityParameterName, "The validation quality of the best solution in the current run."));
239      Parameters.Add(new LookupParameter<SymbolicClassificationSolution>(BestValidationSolutionParameterName, "The best solution on the validation data found in the current run."));
240      Parameters.Add(new LookupParameter<DoubleValue>(BestSolutionAccuracyTrainingParameterName, "The training accuracy of the best solution."));
241      Parameters.Add(new LookupParameter<DoubleValue>(BestSolutionAccuracyTestParameterName, "The test accuracy of the best solution."));
242      Parameters.Add(new LookupParameter<IntValue>(BestSolutionLengthParameterName, "The length of the best symbolic classification solution."));
243      Parameters.Add(new LookupParameter<IntValue>(BestSolutionHeightParameterName, "The height of the best symbolic classification solution."));
244    }
245
246    [StorableHook(HookType.AfterDeserialization)]
247    private void AfterDeserialization() {
248      if (!Parameters.ContainsKey(CalculateSolutionComplexityParameterName)) {
249        Parameters.Add(new ValueParameter<BoolValue>(CalculateSolutionComplexityParameterName, "Determines if the length and height of the validation best solution should be calculated.", new BoolValue(true)));
250      }
251      if (!Parameters.ContainsKey(BestSolutionLengthParameterName)) {
252        Parameters.Add(new LookupParameter<IntValue>(BestSolutionLengthParameterName, "The length of the best symbolic classification solution."));
253      }
254      if (!Parameters.ContainsKey(BestSolutionHeightParameterName)) {
255        Parameters.Add(new LookupParameter<IntValue>(BestSolutionHeightParameterName, "The height of the best symbolic classification solution."));
256      }
257      if (!Parameters.ContainsKey(ApplyLinearScalingParameterName)) {
258        Parameters.Add(new ValueLookupParameter<BoolValue>(ApplyLinearScalingParameterName, "The switch determines if the best solution should be linearly scaled on the whole training set.", new BoolValue(false)));
259      }
260    }
261
262    public override IDeepCloneable Clone(Cloner cloner) {
263      return new ValidationBestSymbolicClassificationSolutionAnalyzer(this, cloner);
264    }
265
266    public override IOperation Apply() {
267      var trees = SymbolicExpressionTree;
268      string targetVariable = ClassificationProblemData.TargetVariable.Value;
269
270      // select a random subset of rows in the validation set
271      int validationStart = ValidiationSamplesStart.Value;
272      int validationEnd = ValidationSamplesEnd.Value;
273      int seed = Random.Next();
274      int count = (int)((validationEnd - validationStart) * RelativeNumberOfEvaluatedSamples.Value);
275      if (count == 0) count = 1;
276      IEnumerable<int> rows = RandomEnumerable.SampleRandomNumbers(seed, validationStart, validationEnd, count)
277         .Where(row => row < ClassificationProblemData.TestSamplesStart.Value || ClassificationProblemData.TestSamplesEnd.Value <= row);
278
279      double upperEstimationLimit = UpperEstimationLimit != null ? UpperEstimationLimit.Value : double.PositiveInfinity;
280      double lowerEstimationLimit = LowerEstimationLimit != null ? LowerEstimationLimit.Value : double.NegativeInfinity;
281
282      double bestQuality = Maximization.Value ? double.NegativeInfinity : double.PositiveInfinity;
283      SymbolicExpressionTree bestTree = null;
284
285      foreach (var tree in trees) {
286        double quality = Evaluator.Evaluate(SymbolicExpressionTreeInterpreter, tree,
287          lowerEstimationLimit, upperEstimationLimit, ClassificationProblemData.Dataset,
288          targetVariable, rows);
289
290        if ((Maximization.Value && quality > bestQuality) ||
291            (!Maximization.Value && quality < bestQuality)) {
292          bestQuality = quality;
293          bestTree = tree;
294        }
295      }
296
297      // if the best validation tree is better than the current best solution => update
298      bool newBest =
299        BestValidationQuality == null ||
300        (Maximization.Value && bestQuality > BestValidationQuality.Value) ||
301        (!Maximization.Value && bestQuality < BestValidationQuality.Value);
302      if (newBest) {
303        if (ApplyLinearScaling.Value) {
304          double alpha, beta;
305          SymbolicRegressionScaledMeanSquaredErrorEvaluator.Calculate(SymbolicExpressionTreeInterpreter, bestTree,
306            lowerEstimationLimit, upperEstimationLimit,
307            ClassificationProblemData.Dataset, targetVariable,
308            ClassificationProblemData.TrainingIndizes, out beta, out alpha);
309
310          // scale tree for solution
311          bestTree = SymbolicRegressionSolutionLinearScaler.Scale(bestTree, alpha, beta);
312        }
313        var model = new SymbolicRegressionModel((ISymbolicExpressionTreeInterpreter)SymbolicExpressionTreeInterpreter.Clone(),
314          bestTree);
315
316        if (BestValidationSolution == null) {
317          BestValidationSolution = new SymbolicClassificationSolution(ClassificationProblemData, model, LowerEstimationLimit.Value, UpperEstimationLimit.Value);
318          BestValidationSolution.Name = BestValidationSolutionParameterName;
319          BestValidationSolution.Description = "Best solution on validation partition found over the whole run.";
320          BestValidationQuality = new DoubleValue(bestQuality);
321        } else {
322          BestValidationSolution.Model = model;
323          BestValidationQuality.Value = bestQuality;
324        }
325
326        UpdateBestSolutionResults();
327      }
328      return base.Apply();
329    }
330
331    private void UpdateBestSolutionResults() {
332      if (CalculateSolutionComplexity.Value) {
333        BestSolutionLength = new IntValue(BestValidationSolution.Model.SymbolicExpressionTree.Size);
334        BestSolutionHeight = new IntValue(BestValidationSolution.Model.SymbolicExpressionTree.Height);
335        if (!Results.ContainsKey(BestSolutionLengthParameterName)) {
336          Results.Add(new Result(BestSolutionLengthParameterName, "Length of the best solution on the validation set", new IntValue()));
337          Results.Add(new Result(BestSolutionHeightParameterName, "Height of the best solution on the validation set", new IntValue()));
338        }
339        Results[BestSolutionLengthParameterName].Value = BestSolutionLength;
340        Results[BestSolutionHeightParameterName].Value = BestSolutionHeight;
341      }
342
343      BestSymbolicRegressionSolutionAnalyzer.UpdateBestSolutionResults(BestValidationSolution, ClassificationProblemData, Results, Generations, VariableFrequencies);
344
345      IEnumerable<double> trainingValues = ClassificationProblemData.Dataset.GetEnumeratedVariableValues(
346        ClassificationProblemData.TargetVariable.Value, ClassificationProblemData.TrainingIndizes);
347      IEnumerable<double> testValues = ClassificationProblemData.Dataset.GetEnumeratedVariableValues(
348        ClassificationProblemData.TargetVariable.Value, ClassificationProblemData.TestIndizes);
349
350      OnlineAccuracyEvaluator accuracyEvaluator = new OnlineAccuracyEvaluator();
351      var originalEnumerator = trainingValues.GetEnumerator();
352      var estimatedEnumerator = BestValidationSolution.EstimatedTrainingClassValues.GetEnumerator();
353      while (originalEnumerator.MoveNext() & estimatedEnumerator.MoveNext()) {
354        accuracyEvaluator.Add(originalEnumerator.Current, estimatedEnumerator.Current);
355      }
356      double trainingAccuracy = accuracyEvaluator.Accuracy;
357
358      accuracyEvaluator.Reset();
359      originalEnumerator = testValues.GetEnumerator();
360      estimatedEnumerator = BestValidationSolution.EstimatedTestClassValues.GetEnumerator();
361      while (originalEnumerator.MoveNext() & estimatedEnumerator.MoveNext()) {
362        accuracyEvaluator.Add(originalEnumerator.Current, estimatedEnumerator.Current);
363      }
364      double testAccuracy = accuracyEvaluator.Accuracy;
365
366      if (!Results.ContainsKey(BestSolutionAccuracyTrainingParameterName)) {
367        BestSolutionAccuracyTraining = new DoubleValue(trainingAccuracy);
368        BestSolutionAccuracyTest = new DoubleValue(testAccuracy);
369
370        Results.Add(new Result(BestSolutionAccuracyTrainingParameterName, BestSolutionAccuracyTraining));
371        Results.Add(new Result(BestSolutionAccuracyTestParameterName, BestSolutionAccuracyTest));
372      } else {
373        BestSolutionAccuracyTraining.Value = trainingAccuracy;
374        BestSolutionAccuracyTest.Value = testAccuracy;
375      }
376    }
377  }
378}
Note: See TracBrowser for help on using the repository browser.