source: trunk/HeuristicLab.Problems.DataAnalysis.Symbolic.Classification/3.4/SingleObjective/SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator.cs @ 16788

Last change on this file since 16788 was 16788, checked in by mkommend, 7 months ago

#2361: Merged all changes from the sensitivity branch to trunk.

File size: 9.7 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Linq;
25using HEAL.Attic;
26using HeuristicLab.Common;
27using HeuristicLab.Core;
28using HeuristicLab.Data;
29using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
30using HeuristicLab.Parameters;
31using HeuristicLab.PluginInfrastructure;
32
33namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Classification {
34  [NonDiscoverableType]
35  [Item("Weighted Performance Measures Evaluator", "Calculates the quality of a symbolic classification solution based on three weighted measures(normalized mean squared error, false negative rate(1-sensitivity) and false positve rate(1-specificity)).")]
36  [StorableType("0772F316-5E12-4153-857E-8625069B4677")]
37  public class SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator : SymbolicClassificationSingleObjectiveEvaluator {
38    private const string NormalizedMeanSquaredErrorWeightingFactorParameterName = "NormalizedMeanSquaredErrorWeightingFactor";
39    private const string FalseNegativeRateWeightingFactorParameterName = "FalseNegativeRateWeightingFactor";
40    private const string FalsePositiveRateWeightingFactorParameterName = "FalsePositiveRateWeightingFactor";
41    private const string ModelCreatorParameterName = "ModelCreator";
42
43    public override bool Maximization { get { return false; } }
44
45    #region parameter properties
46    public IFixedValueParameter<DoubleValue> NormalizedMeanSquaredErrorWeightingFactorParameter {
47      get { return (IFixedValueParameter<DoubleValue>)Parameters[NormalizedMeanSquaredErrorWeightingFactorParameterName]; }
48    }
49    public IFixedValueParameter<DoubleValue> FalseNegativeRateWeightingFactorParameter {
50      get { return (IFixedValueParameter<DoubleValue>)Parameters[FalseNegativeRateWeightingFactorParameterName]; }
51    }
52    public IFixedValueParameter<DoubleValue> FalsePositiveRateWeightingFactorParameter {
53      get { return (IFixedValueParameter<DoubleValue>)Parameters[FalsePositiveRateWeightingFactorParameterName]; }
54    }
55    public IValueLookupParameter<ISymbolicClassificationModelCreator> ModelCreatorParameter {
56      get { return (IValueLookupParameter<ISymbolicClassificationModelCreator>)Parameters[ModelCreatorParameterName]; }
57    }
58    #endregion
59
60    public double NormalizedMeanSquaredErrorWeightingFactor {
61      get { return NormalizedMeanSquaredErrorWeightingFactorParameter.Value.Value; }
62    }
63    public double FalseNegativeRateWeightingFactor {
64      get { return FalseNegativeRateWeightingFactorParameter.Value.Value; }
65    }
66    public double FalsePositiveRateWeightingFactor {
67      get { return FalsePositiveRateWeightingFactorParameter.Value.Value; }
68    }
69
70    [StorableConstructor]
71    protected SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator(StorableConstructorFlag _) : base(_) { }
72    protected SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator(SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator original, Cloner cloner)
73      : base(original, cloner) {
74    }
75    public override IDeepCloneable Clone(Cloner cloner) {
76      return new SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator(this, cloner);
77    }
78
79    public SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator()
80      : base() {
81      Parameters.Add(new FixedValueParameter<DoubleValue>(NormalizedMeanSquaredErrorWeightingFactorParameterName, "The weighting factor of the normalized mean squared error.", new DoubleValue(1)));
82      Parameters.Add(new FixedValueParameter<DoubleValue>(FalseNegativeRateWeightingFactorParameterName, "The weighting factor of the false negative rate (1-sensitivity).", new DoubleValue(1)));
83      Parameters.Add(new FixedValueParameter<DoubleValue>(FalsePositiveRateWeightingFactorParameterName, "The weighting factor of the false positive rate (1-specificity).", new DoubleValue(1)));
84      Parameters.Add(new ValueLookupParameter<ISymbolicClassificationModelCreator>(ModelCreatorParameterName, "The model creator which is used during the evaluations."));
85    }
86
87    public override IOperation InstrumentedApply() {
88      IEnumerable<int> rows = GenerateRowsToEvaluate();
89      var tree = SymbolicExpressionTreeParameter.ActualValue;
90      var creator = ModelCreatorParameter.ActualValue;
91      var interpreter = SymbolicDataAnalysisTreeInterpreterParameter.ActualValue;
92      var estimationLimits = EstimationLimitsParameter.ActualValue;
93      var applyLinearScaling = ApplyLinearScalingParameter.ActualValue.Value;
94
95
96      double quality = Calculate(interpreter, tree, estimationLimits.Lower, estimationLimits.Upper,
97              ProblemDataParameter.ActualValue, rows, applyLinearScaling, creator, NormalizedMeanSquaredErrorWeightingFactor, FalseNegativeRateWeightingFactor, FalsePositiveRateWeightingFactor);
98      QualityParameter.ActualValue = new DoubleValue(quality);
99      return base.InstrumentedApply();
100    }
101
102    public static double Calculate(ISymbolicDataAnalysisExpressionTreeInterpreter interpreter, ISymbolicExpressionTree tree, double lowerEstimationLimit, double upperEstimationLimit, IClassificationProblemData problemData,
103                IEnumerable<int> rows, bool applyLinearScaling, ISymbolicClassificationModelCreator modelCreator, double normalizedMeanSquaredErrorWeightingFactor, double falseNegativeRateWeightingFactor, double falsePositiveRateWeightingFactor) {
104      var estimatedValues = interpreter.GetSymbolicExpressionTreeValues(tree, problemData.Dataset, rows);
105      var targetClassValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows);
106      var boundedEstimatedValues = estimatedValues.LimitToRange(lowerEstimationLimit, upperEstimationLimit).ToArray();
107      OnlineCalculatorError errorState;
108      double nmse;
109
110      //calculate performance measures
111      string positiveClassName = problemData.PositiveClass;
112      double[] classValues, thresholds;
113      IEnumerable<double> estimatedClassValues = null;
114      ISymbolicDiscriminantFunctionClassificationModel m;
115
116      var model = modelCreator.CreateSymbolicClassificationModel(problemData.TargetVariable, tree, interpreter, lowerEstimationLimit, upperEstimationLimit);
117      if ((m = model as ISymbolicDiscriminantFunctionClassificationModel) != null) {
118        m.ThresholdCalculator.Calculate(problemData, boundedEstimatedValues, targetClassValues, out classValues, out thresholds);
119        m.SetThresholdsAndClassValues(thresholds, classValues);
120        estimatedClassValues = m.GetEstimatedClassValues(boundedEstimatedValues);
121      } else {
122        model.RecalculateModelParameters(problemData, rows);
123        estimatedClassValues = model.GetEstimatedClassValues(problemData.Dataset, rows);
124      }
125
126      var performanceCalculator = new ClassificationPerformanceMeasuresCalculator(positiveClassName, problemData.GetClassValue(positiveClassName));
127      performanceCalculator.Calculate(targetClassValues, estimatedClassValues);
128      if (performanceCalculator.ErrorState != OnlineCalculatorError.None)
129        return Double.NaN;
130      double falseNegativeRate = 1 - performanceCalculator.TruePositiveRate;
131      double falsePositiveRate = performanceCalculator.FalsePositiveRate;
132
133      if (applyLinearScaling) {
134        throw new NotSupportedException("The Weighted Performance Measures Evaluator does not suppport linear scaling!");
135      }
136      nmse = OnlineNormalizedMeanSquaredErrorCalculator.Calculate(targetClassValues, boundedEstimatedValues, out errorState);
137      if (errorState != OnlineCalculatorError.None) return Double.NaN;
138      return normalizedMeanSquaredErrorWeightingFactor * nmse + falseNegativeRateWeightingFactor * falseNegativeRate + falsePositiveRateWeightingFactor * falsePositiveRate;
139    }
140
141    public override double Evaluate(IExecutionContext context, ISymbolicExpressionTree tree, IClassificationProblemData problemData, IEnumerable<int> rows) {
142      SymbolicDataAnalysisTreeInterpreterParameter.ExecutionContext = context;
143      EstimationLimitsParameter.ExecutionContext = context;
144      ApplyLinearScalingParameter.ExecutionContext = context;
145      ModelCreatorParameter.ExecutionContext = context;
146
147      double quality = Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, tree, EstimationLimitsParameter.ActualValue.Lower, EstimationLimitsParameter.ActualValue.Upper,
148                                      problemData, rows, ApplyLinearScalingParameter.ActualValue.Value, ModelCreatorParameter.ActualValue, NormalizedMeanSquaredErrorWeightingFactorParameter.Value.Value, FalseNegativeRateWeightingFactor, FalsePositiveRateWeightingFactor);
149
150      SymbolicDataAnalysisTreeInterpreterParameter.ExecutionContext = null;
151      EstimationLimitsParameter.ExecutionContext = null;
152      ApplyLinearScalingParameter.ExecutionContext = null;
153      ModelCreatorParameter.ExecutionContext = null;
154
155      return quality;
156    }
157  }
158}
Note: See TracBrowser for help on using the repository browser.