Changeset 14029 for branches/crossvalidation-2434/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification

Timestamp:

07/08/16 14:40:02 (8 years ago)

Author:

gkronber

Message:

#2434: merged trunk changes r12934:14026 from trunk to branch

Location:

branches/crossvalidation-2434

Files:

• . (modified) (1 prop)
• HeuristicLab.Problems.DataAnalysis (modified) (1 prop)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationEnsembleModel.cs (modified) (5 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationModel.cs (copied) (copied from trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationModel.cs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationPerformanceMeasures.cs (modified) (8 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblem.cs (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblemData.cs (modified) (2 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationSolutionBase.cs (modified) (3 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ConstantClassificationSolution.cs (copied) (copied from trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ConstantClassificationSolution.cs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/DiscriminantFunctionClassificationModel.cs (modified) (4 diffs)

branches/crossvalidation-2434/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationEnsembleModel.cs

@@ -32 +32 @@
   [StorableClass]
   [Item("ClassificationEnsembleModel", "A classification model that contains an ensemble of multiple classification models")]
-  public class ClassificationEnsembleModel : NamedItem, IClassificationEnsembleModel {
+  public class ClassificationEnsembleModel : ClassificationModel, IClassificationEnsembleModel {
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return models.SelectMany(x => x.VariablesUsedForPrediction).Distinct().OrderBy(x => x); }
+    }
 
     [Storable]
@@ -49 +52 @@
     public ClassificationEnsembleModel() : this(Enumerable.Empty<IClassificationModel>()) { }
     public ClassificationEnsembleModel(IEnumerable<IClassificationModel> models)
-      : base() {
+      : base(string.Empty) {
       this.name = ItemName;
       this.description = ItemDescription;
       this.models = new List<IClassificationModel>(models);
+
+      if (this.models.Any()) this.TargetVariable = this.models.First().TargetVariable;
     }
 
@@ -59 +64 @@
     }
 
-    #region IClassificationEnsembleModel Members
     public void Add(IClassificationModel model) {
+      if (string.IsNullOrEmpty(TargetVariable)) TargetVariable = model.TargetVariable;
       models.Add(model);
     }
     public void Remove(IClassificationModel model) {
       models.Remove(model);
+      if (!models.Any()) TargetVariable = string.Empty;
     }
 
@@ -78 +84 @@
     }
 
-    #endregion
 
-    #region IClassificationModel Members
-
-    public IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
       foreach (var estimatedValuesVector in GetEstimatedClassValueVectors(dataset, rows)) {
         // return the class which is most often occuring
@@ -94 +97 @@
     }
 
-    IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
       return new ClassificationEnsembleSolution(models, new ClassificationEnsembleProblemData(problemData));
     }
-    #endregion
+
+
   }
 }

branches/crossvalidation-2434/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationPerformanceMeasures.cs

@@ -37 +37 @@
     protected const string TrainingFalsePositiveRateResultName = "False positive rate (training)";
     protected const string TrainingFalseDiscoveryRateResultName = "False discovery rate (training)";
+    protected const string TrainingF1ScoreResultName = "F1 score (training)";
+    protected const string TrainingMatthewsCorrelationResultName = "Matthews Correlation (training)";
     protected const string TestTruePositiveRateResultName = "True positive rate (test)";
     protected const string TestTrueNegativeRateResultName = "True negative rate (test)";
@@ -43 +45 @@
     protected const string TestFalsePositiveRateResultName = "False positive rate (test)";
     protected const string TestFalseDiscoveryRateResultName = "False discovery rate (test)";
+    protected const string TestF1ScoreResultName = "F1 score (test)";
+    protected const string TestMatthewsCorrelationResultName = "Matthews Correlation (test)";
     #endregion
 
@@ -89 +93 @@
       set { ((DoubleValue)this[TrainingFalseDiscoveryRateResultName].Value).Value = value; }
     }
+    public double TrainingF1Score {
+      get { return ((DoubleValue)this[TrainingF1ScoreResultName].Value).Value; }
+      set { ((DoubleValue)this[TrainingF1ScoreResultName].Value).Value = value; }
+    }
+    public double TrainingMatthewsCorrelation {
+      get { return ((DoubleValue)this[TrainingMatthewsCorrelationResultName].Value).Value; }
+      set { ((DoubleValue)this[TrainingMatthewsCorrelationResultName].Value).Value = value; }
+    }
     public double TestTruePositiveRate {
       get { return ((DoubleValue)this[TestTruePositiveRateResultName].Value).Value; }
@@ -112 +124 @@
       get { return ((DoubleValue)this[TestFalseDiscoveryRateResultName].Value).Value; }
       set { ((DoubleValue)this[TestFalseDiscoveryRateResultName].Value).Value = value; }
+    }
+    public double TestF1Score {
+      get { return ((DoubleValue)this[TestF1ScoreResultName].Value).Value; }
+      set { ((DoubleValue)this[TestF1ScoreResultName].Value).Value = value; }
+    }
+    public double TestMatthewsCorrelation {
+      get { return ((DoubleValue)this[TestMatthewsCorrelationResultName].Value).Value; }
+      set { ((DoubleValue)this[TestMatthewsCorrelationResultName].Value).Value = value; }
     }
     #endregion
@@ -123 +143 @@
       Add(new Result(TrainingFalsePositiveRateResultName, "The false positive rate is the complement of the true negative rate of the model on the training partition.", new PercentValue()));
       Add(new Result(TrainingFalseDiscoveryRateResultName, "The false discovery rate is the complement of the positive predictive value of the model on the training partition.", new PercentValue()));
+      Add(new Result(TrainingF1ScoreResultName, "The F1 score of the model on the training partition.", new DoubleValue()));
+      Add(new Result(TrainingMatthewsCorrelationResultName, "The Matthews correlation value of the model on the training partition.", new DoubleValue()));
       Add(new Result(TestTruePositiveRateResultName, "Sensitivity/True positive rate of the model on the test partition\n(TP/(TP+FN)).", new PercentValue()));
       Add(new Result(TestTrueNegativeRateResultName, "Specificity/True negative rate of the model on the test partition\n(TN/(FP+TN)).", new PercentValue()));
@@ -129 +151 @@
       Add(new Result(TestFalsePositiveRateResultName, "The false positive rate is the complement of the true negative rate of the model on the test partition.", new PercentValue()));
       Add(new Result(TestFalseDiscoveryRateResultName, "The false discovery rate is the complement of the positive predictive value of the model on the test partition.", new PercentValue()));
+      Add(new Result(TestF1ScoreResultName, "The F1 score of the model on the test partition.", new DoubleValue()));
+      Add(new Result(TestMatthewsCorrelationResultName, "The Matthews correlation value of the model on the test partition.", new DoubleValue()));
+
+      Reset();
+    }
+
+
+    public void Reset() {
       TrainingTruePositiveRate = double.NaN;
       TrainingTrueNegativeRate = double.NaN;
@@ -135 +165 @@
       TrainingFalsePositiveRate = double.NaN;
       TrainingFalseDiscoveryRate = double.NaN;
+      TrainingF1Score = double.NaN;
+      TrainingMatthewsCorrelation = double.NaN;
       TestTruePositiveRate = double.NaN;
       TestTrueNegativeRate = double.NaN;
@@ -141 +173 @@
       TestFalsePositiveRate = double.NaN;
       TestFalseDiscoveryRate = double.NaN;
+      TestF1Score = double.NaN;
+      TestMatthewsCorrelation = double.NaN;
     }
 

branches/crossvalidation-2434/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblem.cs

@@ -35 +35 @@
     public override IDeepCloneable Clone(Cloner cloner) { return new ClassificationProblem(this, cloner); }
 
-    public ClassificationProblem()
-      : base() {
-      ProblemData = new ClassificationProblemData();
-    }
+    public ClassificationProblem() : base(new ClassificationProblemData()) { }
   }
 }

branches/crossvalidation-2434/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblemData.cs

@@ -283 +283 @@
     private void AfterDeserialization() {
       RegisterParameterEvents();
+
+      classNamesCache = new List<string>();
+      for (int i = 0; i < ClassNamesParameter.Value.Rows; i++)
+        classNamesCache.Add(ClassNamesParameter.Value[i, 0]);
+
       // BackwardsCompatibility3.4
       #region Backwards compatible code, remove with 3.5
@@ -297 +302 @@
       : base(original, cloner) {
       RegisterParameterEvents();
+      classNamesCache = new List<string>();
+      for (int i = 0; i < ClassNamesParameter.Value.Rows; i++)
+        classNamesCache.Add(ClassNamesParameter.Value[i, 0]);
     }
     public override IDeepCloneable Clone(Cloner cloner) {

branches/crossvalidation-2434/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationSolutionBase.cs

@@ -26 +26 @@
 using HeuristicLab.Optimization;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis.OnlineCalculators;
 
 namespace HeuristicLab.Problems.DataAnalysis {
@@ -128 +129 @@
       TestNormalizedGiniCoefficient = testNormalizedGini;
 
+      ClassificationPerformanceMeasures.Reset();
+
       trainingPerformanceCalculator.Calculate(originalTrainingClassValues, estimatedTrainingClassValues);
       if (trainingPerformanceCalculator.ErrorState == OnlineCalculatorError.None)
@@ -135 +138 @@
       if (testPerformanceCalculator.ErrorState == OnlineCalculatorError.None)
         ClassificationPerformanceMeasures.SetTestResults(testPerformanceCalculator);
+
+      if (ProblemData.Classes == 2) {
+        var f1Training = FOneScoreCalculator.Calculate(originalTrainingClassValues, estimatedTrainingClassValues, out errorState);
+        if (errorState == OnlineCalculatorError.None) ClassificationPerformanceMeasures.TrainingF1Score = f1Training;
+        var f1Test = FOneScoreCalculator.Calculate(originalTestClassValues, estimatedTestClassValues, out errorState);
+        if (errorState == OnlineCalculatorError.None) ClassificationPerformanceMeasures.TestF1Score = f1Test;
+      }
+
+      var mccTraining = MatthewsCorrelationCoefficientCalculator.Calculate(originalTrainingClassValues, estimatedTrainingClassValues, out errorState);
+      if (errorState == OnlineCalculatorError.None) ClassificationPerformanceMeasures.TrainingMatthewsCorrelation = mccTraining;
+      var mccTest = MatthewsCorrelationCoefficientCalculator.Calculate(originalTestClassValues, estimatedTestClassValues, out errorState);
+      if (errorState == OnlineCalculatorError.None) ClassificationPerformanceMeasures.TestMatthewsCorrelation = mccTest;
     }
 

branches/crossvalidation-2434/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/DiscriminantFunctionClassificationModel.cs

@@ -33 +33 @@
   [StorableClass]
   [Item("DiscriminantFunctionClassificationModel", "Represents a classification model that uses a discriminant function and classification thresholds.")]
-  public class DiscriminantFunctionClassificationModel : NamedItem, IDiscriminantFunctionClassificationModel {
+  public class DiscriminantFunctionClassificationModel : ClassificationModel, IDiscriminantFunctionClassificationModel {
+    public override IEnumerable<string> VariablesUsedForPrediction {
+      get { return model.VariablesUsedForPrediction; }
+    }
+
     [Storable]
     private IRegressionModel model;
@@ -73 +77 @@
 
     public DiscriminantFunctionClassificationModel(IRegressionModel model, IDiscriminantFunctionThresholdCalculator thresholdCalculator)
-      : base() {
+      : base(model.TargetVariable) {
       this.name = ItemName;
       this.description = ItemDescription;
+
       this.model = model;
       this.classValues = new double[0];
@@ -115 +120 @@
     }
 
-    public IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+    public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
       if (!Thresholds.Any() && !ClassValues.Any()) throw new ArgumentException("No thresholds and class values were set for the current classification model.");
       foreach (var x in GetEstimatedValues(dataset, rows)) {
@@ -135 +140 @@
     #endregion
 
-    public virtual IDiscriminantFunctionClassificationSolution CreateDiscriminantFunctionClassificationSolution(IClassificationProblemData problemData) {
+    public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return CreateDiscriminantFunctionClassificationSolution(problemData);
+    }
+    public virtual IDiscriminantFunctionClassificationSolution CreateDiscriminantFunctionClassificationSolution(
+      IClassificationProblemData problemData) {
       return new DiscriminantFunctionClassificationSolution(this, new ClassificationProblemData(problemData));
-    }
-
-    public virtual IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
-      return CreateDiscriminantFunctionClassificationSolution(problemData);
     }
   }