Free cookie consent management tool by TermsFeed Policy Generator

Changeset 11622


Ignore:
Timestamp:
12/02/14 17:14:44 (10 years ago)
Author:
ehopf
Message:

#2278 implemented classification quality measures
-true positive rate
-true negative rate
-positive predictive value
-negative predictive value
-false positive rate
-false discovery rate

Location:
branches/Classification-Extensions/HeuristicLab.Problems.DataAnalysis/3.4
Files:
1 added
1 edited

Legend:

Unmodified
Added
Removed
  • branches/Classification-Extensions/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationSolutionBase.cs

    r11171 r11622  
    3535    private const string TestNormalizedGiniCoefficientResultName = "Normalized Gini Coefficient (test)";
    3636
     37    private const string TrainingTruePositiveRateResultName = "True positive rate (training)";
     38    private const string TrainingTrueNegativeRateResultName = "True negative rate (training)";
     39    private const string TrainingPositivePredictiveValueResultName = "Positive predictive value (training)";
     40    private const string TrainingNegativePredictiveValueResultName = "Negative predictive value (training)";
     41    private const string TrainingFalsePositiveRateResultName = "False positive rate (training)";
     42    private const string TrainingFalseDiscoveryRateResultName = "False discovery rate (training)";
     43    private const string TestTruePositiveRateResultName = "True positive rate (test)";
     44    private const string TestTrueNegativeRateResultName = "True negative rate (test)";
     45    private const string TestPositivePredictiveValueResultName = "Positive predictive value (test)";
     46    private const string TestNegativePredictiveValueResultName = "Negative predictive value (test)";
     47    private const string TestFalsePositiveRateResultName = "False positive rate (test)";
     48    private const string TestFalseDiscoveryRateResultName = "False discovery rate (test)";
     49    private const string QualityMeasuresResultName = "Classification Quality Measures";
     50
    3751    public new IClassificationModel Model {
    3852      get { return (IClassificationModel)base.Model; }
     
    6276      protected set { ((DoubleValue)this[TestNormalizedGiniCoefficientResultName].Value).Value = value; }
    6377    }
     78
     79    #region Quality Measures
     80    public ResultCollection QualityMeasures {
     81      get { return ((ResultCollection)this[QualityMeasuresResultName].Value); }
     82      protected set { (this[QualityMeasuresResultName].Value) = value; }
     83    }
     84
     85    public double TrainingTruePositiveRate {
     86      get { return ((DoubleValue)QualityMeasures[TrainingTruePositiveRateResultName].Value).Value; }
     87      protected set { ((DoubleValue)QualityMeasures[TrainingTruePositiveRateResultName].Value).Value = value; }
     88    }
     89    public double TrainingTrueNegativeRate {
     90      get { return ((DoubleValue)QualityMeasures[TrainingTrueNegativeRateResultName].Value).Value; }
     91      protected set { ((DoubleValue)QualityMeasures[TrainingTrueNegativeRateResultName].Value).Value = value; }
     92    }
     93    public double TrainingPositivePredictiveValue {
     94      get { return ((DoubleValue)QualityMeasures[TrainingPositivePredictiveValueResultName].Value).Value; }
     95      protected set { ((DoubleValue)QualityMeasures[TrainingPositivePredictiveValueResultName].Value).Value = value; }
     96    }
     97    public double TrainingNegativePredictiveValue {
     98      get { return ((DoubleValue)QualityMeasures[TrainingNegativePredictiveValueResultName].Value).Value; }
     99      protected set { ((DoubleValue)QualityMeasures[TrainingNegativePredictiveValueResultName].Value).Value = value; }
     100    }
     101    public double TrainingFalsePositiveRate {
     102      get { return ((DoubleValue)QualityMeasures[TrainingFalsePositiveRateResultName].Value).Value; }
     103      protected set { ((DoubleValue)QualityMeasures[TrainingFalsePositiveRateResultName].Value).Value = value; }
     104    }
     105    public double TrainingFalseDiscoveryRate {
     106      get { return ((DoubleValue)QualityMeasures[TrainingFalseDiscoveryRateResultName].Value).Value; }
     107      protected set { ((DoubleValue)QualityMeasures[TrainingFalseDiscoveryRateResultName].Value).Value = value; }
     108    }
     109
     110    public double TestTruePositiveRate {
     111      get { return ((DoubleValue)QualityMeasures[TestTruePositiveRateResultName].Value).Value; }
     112      protected set { ((DoubleValue)QualityMeasures[TestTruePositiveRateResultName].Value).Value = value; }
     113    }
     114    public double TestTrueNegativeRate {
     115      get { return ((DoubleValue)QualityMeasures[TestTrueNegativeRateResultName].Value).Value; }
     116      protected set { ((DoubleValue)QualityMeasures[TestTrueNegativeRateResultName].Value).Value = value; }
     117    }
     118    public double TestPositivePredictiveValue {
     119      get { return ((DoubleValue)QualityMeasures[TestPositivePredictiveValueResultName].Value).Value; }
     120      protected set { ((DoubleValue)QualityMeasures[TestPositivePredictiveValueResultName].Value).Value = value; }
     121    }
     122    public double TestNegativePredictiveValue {
     123      get { return ((DoubleValue)QualityMeasures[TestNegativePredictiveValueResultName].Value).Value; }
     124      protected set { ((DoubleValue)QualityMeasures[TestNegativePredictiveValueResultName].Value).Value = value; }
     125    }
     126    public double TestFalsePositiveRate {
     127      get { return ((DoubleValue)QualityMeasures[TestFalsePositiveRateResultName].Value).Value; }
     128      protected set { ((DoubleValue)QualityMeasures[TestFalsePositiveRateResultName].Value).Value = value; }
     129    }
     130    public double TestFalseDiscoveryRate {
     131      get { return ((DoubleValue)QualityMeasures[TestFalseDiscoveryRateResultName].Value).Value; }
     132      protected set { ((DoubleValue)QualityMeasures[TestFalseDiscoveryRateResultName].Value).Value = value; }
     133    }
     134    #endregion
    64135    #endregion
    65136
     
    75146      Add(new Result(TrainingNormalizedGiniCoefficientResultName, "Normalized Gini coefficient of the model on the training partition.", new DoubleValue()));
    76147      Add(new Result(TestNormalizedGiniCoefficientResultName, "Normalized Gini coefficient of the model on the test partition.", new DoubleValue()));
     148      AddQualityMeasuresResultCollection();
    77149    }
    78150
     
    83155      if (!this.ContainsKey(TestNormalizedGiniCoefficientResultName))
    84156        Add(new Result(TestNormalizedGiniCoefficientResultName, "Normalized Gini coefficient of the model on the test partition.", new DoubleValue()));
     157      if (!this.ContainsKey(QualityMeasuresResultName))
     158        AddQualityMeasuresResultCollection();
     159    }
     160
     161    protected void AddQualityMeasuresResultCollection() {
     162      ResultCollection qualityMeasuresResult = new ResultCollection();
     163      qualityMeasuresResult.Add(new Result(TrainingTruePositiveRateResultName, "Sensitivity/True positive rate of the model on the training partition\n(TP/(TP+FN)).", new PercentValue()));
     164      qualityMeasuresResult.Add(new Result(TrainingTrueNegativeRateResultName, "Specificity/True negative rate of the model on the training partition\n(TN/(FP+TN)).", new PercentValue()));
     165      qualityMeasuresResult.Add(new Result(TrainingPositivePredictiveValueResultName, "Precision/Positive predictive value of the model on the training partition\n(TP/(TP+FP)).", new PercentValue()));
     166      qualityMeasuresResult.Add(new Result(TrainingNegativePredictiveValueResultName, "Negative predictive value of the model on the training partition\n(TN/(TN+FN)).", new PercentValue()));
     167      qualityMeasuresResult.Add(new Result(TrainingFalsePositiveRateResultName, "The false positive rate is the complement of the true negative rate of the model on the training partition.", new PercentValue()));
     168      qualityMeasuresResult.Add(new Result(TrainingFalseDiscoveryRateResultName, "The false discovery rate is the complement of the positive predictive value of the model on the training partition.", new PercentValue()));
     169      qualityMeasuresResult.Add(new Result(TestTruePositiveRateResultName, "Sensitivity/True positive rate of the model on the test partition\n(TP/(TP+FN)).", new PercentValue()));
     170      qualityMeasuresResult.Add(new Result(TestTrueNegativeRateResultName, "Specificity/True negative rate of the model on the test partition\n(TN/(FP+TN)).", new PercentValue()));
     171      qualityMeasuresResult.Add(new Result(TestPositivePredictiveValueResultName, "Precision/Positive predictive value of the model on the test partition\n(TP/(TP+FP)).", new PercentValue()));
     172      qualityMeasuresResult.Add(new Result(TestNegativePredictiveValueResultName, "Negative predictive value of the model on the test partition\n(TN/(TN+FN)).", new PercentValue()));
     173      qualityMeasuresResult.Add(new Result(TestFalsePositiveRateResultName, "The false positive rate is the complement of the true negative rate of the model on the test partition.", new PercentValue()));
     174      qualityMeasuresResult.Add(new Result(TestFalseDiscoveryRateResultName, "The false discovery rate is the complement of the positive predictive value of the model on the test partition.", new PercentValue()));
     175      Add(new Result(QualityMeasuresResultName, "Classification quality measures.\nIn Multiclass Classification all misclassifications of the negative class will be treated as true negatives.", qualityMeasuresResult));
    85176    }
    86177
     
    88179      double[] estimatedTrainingClassValues = EstimatedTrainingClassValues.ToArray(); // cache values
    89180      double[] originalTrainingClassValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndices).ToArray();
     181
    90182      double[] estimatedTestClassValues = EstimatedTestClassValues.ToArray(); // cache values
    91183      double[] originalTestClassValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TestIndices).ToArray();
     184
     185      var positiveClassName = ProblemData.PositiveClassName;
     186      double positiveClassValue = ProblemData.GetClassValue(positiveClassName);
     187      QualityCalculator trainingQualityCalculator = new QualityCalculator(positiveClassValue);
     188      QualityCalculator testQualityCalculator = new QualityCalculator(positiveClassValue);
    92189
    93190      OnlineCalculatorError errorState;
     
    107204      TrainingNormalizedGiniCoefficient = trainingNormalizedGini;
    108205      TestNormalizedGiniCoefficient = testNormalizedGini;
     206
     207      //quality measures training partition
     208      trainingQualityCalculator.Calculate(originalTrainingClassValues, estimatedTrainingClassValues, out errorState);
     209      if (errorState != OnlineCalculatorError.None) {
     210        TrainingTruePositiveRate = double.NaN;
     211        TrainingTrueNegativeRate = double.NaN;
     212        TrainingPositivePredictiveValue = double.NaN;
     213        TrainingNegativePredictiveValue = double.NaN;
     214        TrainingFalsePositiveRate = double.NaN;
     215        TrainingFalseDiscoveryRate = double.NaN;
     216      } else {
     217        TrainingTruePositiveRate = trainingQualityCalculator.TruePositiveRate;
     218        TrainingTrueNegativeRate = trainingQualityCalculator.TrueNegativeRate;
     219        TrainingPositivePredictiveValue = trainingQualityCalculator.PositivePredictiveValue;
     220        TrainingNegativePredictiveValue = trainingQualityCalculator.NegativePredictiveValue;
     221        TrainingFalsePositiveRate = trainingQualityCalculator.FalsePositiveRate;
     222        TrainingFalseDiscoveryRate = trainingQualityCalculator.FalseDiscoveryRate;
     223      }
     224      //quality measures test partition
     225      testQualityCalculator.Calculate(originalTestClassValues, estimatedTestClassValues, out errorState);
     226      if (errorState != OnlineCalculatorError.None) {
     227        TestTruePositiveRate = double.NaN;
     228        TestTrueNegativeRate = double.NaN;
     229        TestPositivePredictiveValue = double.NaN;
     230        TestNegativePredictiveValue = double.NaN;
     231        TestFalsePositiveRate = double.NaN;
     232        TestFalseDiscoveryRate = double.NaN;
     233      } else {
     234        TestTruePositiveRate = testQualityCalculator.TruePositiveRate;
     235        TestTrueNegativeRate = testQualityCalculator.TrueNegativeRate;
     236        TestPositivePredictiveValue = testQualityCalculator.PositivePredictiveValue;
     237        TestNegativePredictiveValue = testQualityCalculator.NegativePredictiveValue;
     238        TestFalsePositiveRate = testQualityCalculator.FalsePositiveRate;
     239        TestFalseDiscoveryRate = testQualityCalculator.FalseDiscoveryRate;
     240      }
    109241    }
    110242
Note: See TracChangeset for help on using the changeset viewer.