source: branches/SensitivityEvaluator/HeuristicLab.Problems.DataAnalysis.Symbolic.Classification/3.4/SingleObjective/SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator.cs @ 12487

Last change on this file since 12487 was 12487, checked in by ehopf, 5 years ago

#2361: Fixed a bug that causes an exception when a ModelCreator is used that doesn´t implement the ISymbolicDiscriminantFunctionClassificationModelCreator interface. Note that the runtime can be further improved for those ModelCreators because of redundant calculations.

File size: 10.1 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Linq;
25using HeuristicLab.Common;
26using HeuristicLab.Core;
27using HeuristicLab.Data;
28using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
29using HeuristicLab.Parameters;
30using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
31
32namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Classification.SingleObjective {
33  [Item("Weighted Performance Measures Evaluator", "Calculates the quality of a symbolic classification solution based on three weighted measures(normalized mean squared error, false negative rate(1-sensitivity) and false positve rate(1-specificity)).")]
34  [StorableClass]
35  public class SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator : SymbolicClassificationSingleObjectiveEvaluator {
36    private const string NormalizedMeanSquaredErrorWeightingFactorParameterName = "NormalizedMeanSquaredErrorWeightingFactor";
37    private const string FalseNegativeRateWeightingFactorParameterName = "FalseNegativeRateWeightingFactor";
38    private const string FalsePositiveRateWeightingFactorParameterName = "FalsePositiveRateWeightingFactor";
39    private const string ModelCreatorParameterName = "ModelCreator";
40
41    public override bool Maximization { get { return false; } }
42
43    #region parameter properties
44    public IFixedValueParameter<DoubleValue> NormalizedMeanSquaredErrorWeightingFactorParameter {
45      get { return (IFixedValueParameter<DoubleValue>)Parameters[NormalizedMeanSquaredErrorWeightingFactorParameterName]; }
46    }
47    public IFixedValueParameter<DoubleValue> FalseNegativeRateWeightingFactorParameter {
48      get { return (IFixedValueParameter<DoubleValue>)Parameters[FalseNegativeRateWeightingFactorParameterName]; }
49    }
50    public IFixedValueParameter<DoubleValue> FalsePositiveRateWeightingFactorParameter {
51      get { return (IFixedValueParameter<DoubleValue>)Parameters[FalsePositiveRateWeightingFactorParameterName]; }
52    }
53    public IValueLookupParameter<ISymbolicClassificationModelCreator> ModelCreatorParameter {
54      get { return (IValueLookupParameter<ISymbolicClassificationModelCreator>)Parameters[ModelCreatorParameterName]; }
55    }
56    #endregion
57
58    public double NormalizedMeanSquaredErrorWeightingFactor {
59      get {
60        return NormalizedMeanSquaredErrorWeightingFactorParameter.Value.Value;
61      }
62    }
63    public double FalseNegativeRateWeightingFactor {
64      get {
65        return FalseNegativeRateWeightingFactorParameter.Value.Value;
66      }
67    }
68    public double FalsePositiveRateWeightingFactor {
69      get {
70        return FalsePositiveRateWeightingFactorParameter.Value.Value;
71      }
72    }
73
74    [StorableConstructor]
75    protected SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator(bool deserializing) : base(deserializing) { }
76    protected SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator(SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator original, Cloner cloner)
77      : base(original, cloner) {
78    }
79    public override IDeepCloneable Clone(Cloner cloner) {
80      return new SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator(this, cloner);
81    }
82
83    [StorableHook(HookType.AfterDeserialization)]
84    private void AfterDeserialization() {
85      // BackwardsCompatibility
86      #region Backwards compatible code
87      if (Parameters.ContainsKey(ModelCreatorParameterName)) {
88        Parameters.Remove(ModelCreatorParameterName);
89        Parameters.Add(new ValueLookupParameter<ISymbolicClassificationModelCreator>(ModelCreatorParameterName, "The model creator which is used during the evaluations."));
90      }
91      #endregion
92    }
93
94    public SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator()
95      : base() {
96      Parameters.Add(new FixedValueParameter<DoubleValue>(NormalizedMeanSquaredErrorWeightingFactorParameterName, "The weighting factor of the normalized mean squared error.", new DoubleValue(1)));
97      Parameters.Add(new FixedValueParameter<DoubleValue>(FalseNegativeRateWeightingFactorParameterName, "The weighting factor of the false negative rate (1-sensitivity).", new DoubleValue(1)));
98      Parameters.Add(new FixedValueParameter<DoubleValue>(FalsePositiveRateWeightingFactorParameterName, "The weighting factor of the false positive rate (1-specificity).", new DoubleValue(1)));
99      Parameters.Add(new ValueLookupParameter<ISymbolicClassificationModelCreator>(ModelCreatorParameterName, "The model creator which is used during the evaluations."));
100    }
101
102    public override IOperation InstrumentedApply() {
103      IEnumerable<int> rows = GenerateRowsToEvaluate();
104      var solution = SymbolicExpressionTreeParameter.ActualValue;
105      var creator = ModelCreatorParameter.ActualValue;
106      double quality = Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, solution, EstimationLimitsParameter.ActualValue.Lower, EstimationLimitsParameter.ActualValue.Upper,
107              ProblemDataParameter.ActualValue, rows, ApplyLinearScalingParameter.ActualValue.Value, creator, NormalizedMeanSquaredErrorWeightingFactor, FalseNegativeRateWeightingFactor, FalsePositiveRateWeightingFactor);
108      QualityParameter.ActualValue = new DoubleValue(quality);
109      return base.InstrumentedApply();
110    }
111
112    public static double Calculate(ISymbolicDataAnalysisExpressionTreeInterpreter interpreter, ISymbolicExpressionTree solution, double lowerEstimationLimit, double upperEstimationLimit, IClassificationProblemData problemData,
113                IEnumerable<int> rows, bool applyLinearScaling, ISymbolicClassificationModelCreator modelCreator, double normalizedMeanSquaredErrorWeightingFactor, double falseNegativeRateWeightingFactor, double falsePositiveRateWeightingFactor) {
114      IEnumerable<double> estimatedValues = interpreter.GetSymbolicExpressionTreeValues(solution, problemData.Dataset, rows);
115      var targetClassValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows);
116      var boundedEstimatedValues = estimatedValues.LimitToRange(lowerEstimationLimit, upperEstimationLimit).ToArray();
117      OnlineCalculatorError errorState;
118      double nmse;
119
120      //calculate performance measures
121      string positiveClassName = problemData.PositiveClass;
122      double[] classValues, thresholds;
123      IEnumerable<double> estimatedClassValues = null;
124      ISymbolicDiscriminantFunctionClassificationModel m;
125
126      var model = modelCreator.CreateSymbolicClassificationModel(solution, interpreter, lowerEstimationLimit, upperEstimationLimit);
127      if ((m = model as ISymbolicDiscriminantFunctionClassificationModel) != null) {
128        m.ThresholdCalculator.Calculate(problemData, boundedEstimatedValues, targetClassValues, out classValues, out thresholds);
129        m.SetThresholdsAndClassValues(thresholds, classValues);
130        estimatedClassValues = m.GetEstimatedClassValues(boundedEstimatedValues);
131      } else {
132        model.RecalculateModelParameters(problemData, rows);
133        estimatedClassValues = model.GetEstimatedClassValues(problemData.Dataset, rows);
134      }
135      var performanceCalculator = new ClassificationPerformanceMeasuresCalculator(positiveClassName, problemData.GetClassValue(positiveClassName));
136      performanceCalculator.Calculate(targetClassValues, estimatedClassValues);
137      if (performanceCalculator.ErrorState != OnlineCalculatorError.None)
138        return Double.NaN;
139      double falseNegativeRate = 1 - performanceCalculator.TruePositiveRate;
140      double falsePositiveRate = performanceCalculator.FalsePositiveRate;
141
142      if (applyLinearScaling) {
143        throw new NotSupportedException("The Weighted Performance Measures Evaluator does not suppport linear scaling!");
144      }
145      nmse = OnlineNormalizedMeanSquaredErrorCalculator.Calculate(targetClassValues, boundedEstimatedValues, out errorState);
146      if (errorState != OnlineCalculatorError.None) return Double.NaN;
147      return normalizedMeanSquaredErrorWeightingFactor * nmse + falseNegativeRateWeightingFactor * falseNegativeRate + falsePositiveRateWeightingFactor * falsePositiveRate;
148    }
149
150    public override double Evaluate(IExecutionContext context, ISymbolicExpressionTree tree, IClassificationProblemData problemData, IEnumerable<int> rows) {
151      SymbolicDataAnalysisTreeInterpreterParameter.ExecutionContext = context;
152      EstimationLimitsParameter.ExecutionContext = context;
153      ApplyLinearScalingParameter.ExecutionContext = context;
154      ModelCreatorParameter.ExecutionContext = context;
155
156      double quality = Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, tree, EstimationLimitsParameter.ActualValue.Lower, EstimationLimitsParameter.ActualValue.Upper,
157                                      problemData, rows, ApplyLinearScalingParameter.ActualValue.Value, ModelCreatorParameter.ActualValue, NormalizedMeanSquaredErrorWeightingFactorParameter.Value.Value, FalseNegativeRateWeightingFactor, FalsePositiveRateWeightingFactor);
158
159      SymbolicDataAnalysisTreeInterpreterParameter.ExecutionContext = null;
160      EstimationLimitsParameter.ExecutionContext = null;
161      ApplyLinearScalingParameter.ExecutionContext = null;
162      ModelCreatorParameter.ExecutionContext = null;
163
164      return quality;
165    }
166  }
167}
Note: See TracBrowser for help on using the repository browser.