Changeset 11872

Timestamp:

02/03/15 14:15:26 (10 years ago)

Author:

mkommend

Message:

#2278: Merged r11762, r11763, r11764 and r11766 into stable.

Location:

stable

Files:

• . (modified) (1 prop)
• HeuristicLab.Optimization (modified) (1 prop)
• HeuristicLab.Optimization/3.3/Algorithms/Algorithm.cs (modified) (1 diff)
• HeuristicLab.Optimization/3.3/ResultCollection.cs (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis (modified) (1 prop)
• HeuristicLab.Problems.DataAnalysis/3.4/HeuristicLab.Problems.DataAnalysis-3.4.csproj (modified) (2 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationPerformanceMeasures.cs (copied) (copied from trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationPerformanceMeasures.cs) (2 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblemData.cs (modified) (9 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationSolutionBase.cs (modified) (6 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Interfaces/Classification/IClassificationProblemData.cs (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis/3.4/OnlineCalculators/ClassificationPerformanceMeasuresCalculator.cs (copied) (copied from trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/OnlineCalculators/ClassificationPerformanceMeasuresCalculator.cs)

stable/HeuristicLab.Optimization/3.3/Algorithms/Algorithm.cs

@@ -227 +227 @@
     public virtual void CollectResultValues(IDictionary<string, IItem> values) {
       values.Add("Execution Time", new TimeSpanValue(ExecutionTime));
-      CollectResultsRecursively("", Results, values);
-    }
-
-    private void CollectResultsRecursively(string path, ResultCollection results, IDictionary<string, IItem> values) {
-      foreach (IResult result in results) {
-        values.Add(path + result.Name, result.Value);
-        ResultCollection childCollection = result.Value as ResultCollection;
-        if (childCollection != null) {
-          CollectResultsRecursively(path + result.Name + ".", childCollection, values);
-        }
-      }
+      Results.CollectResultValues(values);
     }
 

stable/HeuristicLab.Optimization/3.3/ResultCollection.cs

@@ -44 +44 @@
       get { return HeuristicLab.Common.Resources.VSImageLibrary.Object; }
     }
+
+    public virtual void CollectResultValues(IDictionary<string, IItem> values) {
+      CollectResultValues(values, string.Empty);
+    }
+
+    public virtual void CollectResultValues(IDictionary<string, IItem> values, string rootPath) {
+      foreach (IResult result in this) {
+        var children = GetCollectedResults(result);
+        string path = string.Empty;
+        if (!string.IsNullOrWhiteSpace(rootPath))
+          path = rootPath + ".";
+        foreach (var c in children) {
+          if (string.IsNullOrEmpty(c.Key))
+            values.Add(path + result.Name, c.Value);
+          else values.Add(path + result.Name + "." + c.Key, c.Value);
+        }
+      }
+    }
+
+    protected virtual IEnumerable<KeyValuePair<string, IItem>> GetCollectedResults(IResult result) {
+      if (result.Value == null) yield break;
+      yield return new KeyValuePair<string, IItem>(string.Empty, result.Value);
+
+      var resultCollection = result.Value as ResultCollection;
+      if (resultCollection != null) {
+        var children = new Dictionary<string, IItem>();
+        resultCollection.CollectResultValues(children);
+        foreach (var child in children) yield return child;
+      }
+    }
+
   }
 }

stable/HeuristicLab.Problems.DataAnalysis/3.4/HeuristicLab.Problems.DataAnalysis-3.4.csproj

@@ -118 +118 @@
     </Compile>
     <Compile Include="Implementation\Classification\ClassificationEnsembleSolution.cs" />
+    <Compile Include="Implementation\Classification\ClassificationPerformanceMeasures.cs" />
     <Compile Include="Implementation\Classification\ClassificationProblemData.cs" />
     <Compile Include="Implementation\Classification\ClassificationProblem.cs" />
@@ -172 +173 @@
     <Compile Include="Interfaces\TimeSeriesPrognosis\ITimeSeriesPrognosisSolution.cs" />
     <Compile Include="OnlineCalculators\AutoCorrelationCalculator.cs" />
+    <Compile Include="OnlineCalculators\ClassificationPerformanceMeasuresCalculator.cs" />
     <Compile Include="OnlineCalculators\DependencyCalculator\HoeffdingsDependenceCalculator.cs" />
     <Compile Include="OnlineCalculators\DependencyCalculator\PearsonsRDependenceCalculator.cs" />

stable/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationPerformanceMeasures.cs

@@ -21 +21 @@
 
 using System;
+using HeuristicLab.Common;
 using HeuristicLab.Data;
 using HeuristicLab.Optimization;
@@ -51 +52 @@
     protected ClassificationPerformanceMeasuresResultCollection(bool deserializing)
       : base(deserializing) {
+    }
+
+    protected ClassificationPerformanceMeasuresResultCollection(ClassificationPerformanceMeasuresResultCollection original, Cloner cloner)
+      : base(original, cloner) { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new ClassificationPerformanceMeasuresResultCollection(this, cloner);
     }
 

stable/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblemData.cs

@@ -36 +36 @@
     protected const string ClassNamesParameterName = "ClassNames";
     protected const string ClassificationPenaltiesParameterName = "ClassificationPenalties";
+    protected const string PositiveClassParameterName = "PositiveClass";
     protected const int MaximumNumberOfClasses = 100;
     protected const int InspectedRowsToDetermineTargets = 2000;
@@ -213 +214 @@
       get { return (IFixedValueParameter<StringMatrix>)Parameters[ClassNamesParameterName]; }
     }
+    public IConstrainedValueParameter<StringValue> PositiveClassParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[PositiveClassParameterName]; }
+    }
     public IFixedValueParameter<DoubleMatrix> ClassificationPenaltiesParameter {
       get { return (IFixedValueParameter<DoubleMatrix>)Parameters[ClassificationPenaltiesParameterName]; }
@@ -262 +266 @@
       get { return ClassNamesCache; }
     }
+
+    public string PositiveClass {
+      get { return PositiveClassParameter.Value.Value; }
+      set {
+        var matchingValue = PositiveClassParameter.ValidValues.SingleOrDefault(x => x.Value == value);
+        if (matchingValue == null) throw new ArgumentException(string.Format("{0} cannot be set as positive class.", value));
+        PositiveClassParameter.Value = matchingValue;
+      }
+    }
     #endregion
 
@@ -270 +283 @@
     private void AfterDeserialization() {
       RegisterParameterEvents();
+      // BackwardsCompatibility3.4
+      #region Backwards compatible code, remove with 3.5
+      if (!Parameters.ContainsKey(PositiveClassParameterName)) {
+        var validValues = new ItemSet<StringValue>(ClassNames.Select(s => new StringValue(s).AsReadOnly()));
+        Parameters.Add(new ConstrainedValueParameter<StringValue>(PositiveClassParameterName,
+          "The positive class which is used for quality measure calculation (e.g., specifity, sensitivity,...)", validValues, validValues.First()));
+      }
+      #endregion
+
     }
 
@@ -307 +329 @@
       Parameters.Add(new ConstrainedValueParameter<StringValue>(TargetVariableParameterName, new ItemSet<StringValue>(validTargetVariableValues), target));
       Parameters.Add(new FixedValueParameter<StringMatrix>(ClassNamesParameterName, ""));
+      Parameters.Add(new ConstrainedValueParameter<StringValue>(PositiveClassParameterName, "The positive class which is used for quality measure calculation (e.g., specifity, sensitivity,...)"));
       Parameters.Add(new FixedValueParameter<DoubleMatrix>(ClassificationPenaltiesParameterName, ""));
 
@@ -340 +363 @@
       ClassNamesParameter.Value.RowNames = ClassValues.Select(s => "ClassValue: " + s);
 
+      PositiveClassParameter.ValidValues.Clear();
+      foreach (var className in ClassNames) {
+        PositiveClassParameter.ValidValues.Add(new StringValue(className).AsReadOnly());
+      }
+
       ((IStringConvertibleMatrix)ClassificationPenaltiesParameter.Value).Rows = Classes;
       ((IStringConvertibleMatrix)ClassificationPenaltiesParameter.Value).Columns = Classes;
@@ -411 +439 @@
     }
     private void Parameter_ValueChanged(object sender, EventArgs e) {
+      var oldPositiveClass = PositiveClass;
+      var oldClassNames = classNamesCache;
+      var index = oldClassNames.IndexOf(oldPositiveClass);
+
       classNamesCache = null;
       ClassificationPenaltiesParameter.Value.RowNames = ClassNames.Select(name => "Actual " + name);
       ClassificationPenaltiesParameter.Value.ColumnNames = ClassNames.Select(name => "Estimated " + name);
+
+      PositiveClassParameter.ValidValues.Clear();
+      foreach (var className in ClassNames) {
+        PositiveClassParameter.ValidValues.Add(new StringValue(className).AsReadOnly());
+      }
+      PositiveClassParameter.Value = PositiveClassParameter.ValidValues.ElementAt(index);
+
       OnChanged();
     }
@@ -435 +474 @@
       if (!newClassValues.SequenceEqual(ClassValues)) {
         errorMessage = errorMessage + string.Format("The class values differ in the provided classification problem data.");
-        return false;
+        returnValue = false;
+      }
+
+      var newPositivieClassName = classificationProblemData.PositiveClass;
+      if (newPositivieClassName != PositiveClass) {
+        errorMessage = errorMessage + string.Format("The positive class differs in the provided classification problem data.");
+        returnValue = false;
       }
 
@@ -452 +497 @@
         ClassNamesParameter.Value[i, 0] = classificationProblemData.ClassNames.ElementAt(i);
 
+      PositiveClass = classificationProblemData.PositiveClass;
+
       for (int i = 0; i < Classes; i++) {
         for (int j = 0; j < Classes; j++) {

stable/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationSolutionBase.cs

@@ -34 +34 @@
     private const string TrainingNormalizedGiniCoefficientResultName = "Normalized Gini Coefficient (training)";
     private const string TestNormalizedGiniCoefficientResultName = "Normalized Gini Coefficient (test)";
+    private const string ClassificationPerformanceMeasuresResultName = "Classification Performance Measures";
 
     public new IClassificationModel Model {
@@ -62 +63 @@
       protected set { ((DoubleValue)this[TestNormalizedGiniCoefficientResultName].Value).Value = value; }
     }
+    public ClassificationPerformanceMeasuresResultCollection ClassificationPerformanceMeasures {
+      get { return ((ClassificationPerformanceMeasuresResultCollection)this[ClassificationPerformanceMeasuresResultName].Value); }
+      protected set { (this[ClassificationPerformanceMeasuresResultName].Value) = value; }
+    }
     #endregion
 
@@ -75 +80 @@
       Add(new Result(TrainingNormalizedGiniCoefficientResultName, "Normalized Gini coefficient of the model on the training partition.", new DoubleValue()));
       Add(new Result(TestNormalizedGiniCoefficientResultName, "Normalized Gini coefficient of the model on the test partition.", new DoubleValue()));
+      Add(new Result(ClassificationPerformanceMeasuresResultName, @"Classification performance measures.\n
+                              In a multiclass classification all misclassifications of the negative class will be treated as true negatives except on positive class estimations.",
+                            new ClassificationPerformanceMeasuresResultCollection()));
     }
 
@@ -83 +91 @@
       if (!this.ContainsKey(TestNormalizedGiniCoefficientResultName))
         Add(new Result(TestNormalizedGiniCoefficientResultName, "Normalized Gini coefficient of the model on the test partition.", new DoubleValue()));
+      if (!this.ContainsKey(ClassificationPerformanceMeasuresResultName)) {
+        Add(new Result(ClassificationPerformanceMeasuresResultName, @"Classification performance measures.\n
+                              In a multiclass classification all misclassifications of the negative class will be treated as true negatives except on positive class estimations.",
+                              new ClassificationPerformanceMeasuresResultCollection()));
+        CalculateClassificationResults();
+      }
     }
 
@@ -88 +102 @@
       double[] estimatedTrainingClassValues = EstimatedTrainingClassValues.ToArray(); // cache values
       double[] originalTrainingClassValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndices).ToArray();
+
       double[] estimatedTestClassValues = EstimatedTestClassValues.ToArray(); // cache values
       double[] originalTestClassValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TestIndices).ToArray();
+
+      var positiveClassName = ProblemData.PositiveClass;
+      double positiveClassValue = ProblemData.GetClassValue(positiveClassName);
+      ClassificationPerformanceMeasuresCalculator trainingPerformanceCalculator = new ClassificationPerformanceMeasuresCalculator(positiveClassName, positiveClassValue);
+      ClassificationPerformanceMeasuresCalculator testPerformanceCalculator = new ClassificationPerformanceMeasuresCalculator(positiveClassName, positiveClassValue);
 
       OnlineCalculatorError errorState;
@@ -107 +127 @@
       TrainingNormalizedGiniCoefficient = trainingNormalizedGini;
       TestNormalizedGiniCoefficient = testNormalizedGini;
+
+      trainingPerformanceCalculator.Calculate(originalTrainingClassValues, estimatedTrainingClassValues);
+      if (trainingPerformanceCalculator.ErrorState == OnlineCalculatorError.None)
+        ClassificationPerformanceMeasures.SetTrainingResults(trainingPerformanceCalculator);
+
+      testPerformanceCalculator.Calculate(originalTestClassValues, estimatedTestClassValues);
+      if (testPerformanceCalculator.ErrorState == OnlineCalculatorError.None)
+        ClassificationPerformanceMeasures.SetTestResults(testPerformanceCalculator);
     }
 

stable/HeuristicLab.Problems.DataAnalysis/3.4/Interfaces/Classification/IClassificationProblemData.cs

@@ -26 +26 @@
 
     IEnumerable<string> ClassNames { get; }
+    string PositiveClass { get; set; }
     IEnumerable<double> ClassValues { get; }
     int Classes { get; }