Changeset 8554

Timestamp:

09/03/12 13:27:40 (12 years ago)

Author:

mkommend

Message:

#1915:

• Corrected class names and class values caching in ClassificationProblemData
• Removed caching of classification penalties
• Corrected AccuracyMaximizationThresholdCalculator (retrieving of penalties was switched)
• Added asserts for the achieved accuracy to the classification sample unit test

Location:

trunk/sources

Files:

• HeuristicLab.Problems.DataAnalysis (modified) (1 prop)
• HeuristicLab.Problems.DataAnalysis.Views (modified) (1 prop)
• HeuristicLab.Problems.DataAnalysis.Views/3.4/Classification/ClassificationEnsembleSolutionEstimatedClassValuesView.cs (modified) (2 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblemData.cs (modified) (7 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ThresholdCalculators/AccuracyMaximizationThresholdCalculator.cs (modified) (1 diff)
• HeuristicLab.Tests/HeuristicLab-3.3/SamplesTest.cs (modified) (1 diff)

trunk/sources/HeuristicLab.Problems.DataAnalysis.Views/3.4/Classification/ClassificationEnsembleSolutionEstimatedClassValuesView.cs

@@ -96 +96 @@
       }
 
-      int classValuesCount = Content.ProblemData.ClassValues.Count;
+      int classValuesCount = Content.ProblemData.Classes;
       int solutionsCount = Content.ClassificationSolutions.Count();
       string[,] values = new string[indices.Length, 5 + classValuesCount + solutionsCount];
@@ -114 +114 @@
             estimatedValuesVector[i].GroupBy(x => x).Select(g => new { Key = g.Key, Count = g.Count() }).ToList();
           var estimationCount = groups.Where(g => g.Key != null).Select(g => g.Count).Sum();
-          values[i, 4] =
-            (((double)groups.Where(g => g.Key == estimatedClassValues[i]).Single().Count) / estimationCount).ToString();
-          for (int classIndex = 0; classIndex < Content.ProblemData.ClassValues.Count; classIndex++) {
-            var group = groups.Where(g => g.Key == Content.ProblemData.ClassValues[classIndex]).SingleOrDefault();
+          values[i, 4] = (((double)groups.Where(g => g.Key == estimatedClassValues[i]).Single().Count) / estimationCount).ToString();
+          for (int classIndex = 0; classIndex < Content.ProblemData.Classes; classIndex++) {
+            var group = groups.Where(g => g.Key == Content.ProblemData.ClassValues.ElementAt(classIndex)).SingleOrDefault();
             if (group == null) values[i, 5 + classIndex] = 0.ToString();
             else values[i, 5 + classIndex] = group.Count.ToString();

trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblemData.cs

@@ -223 +223 @@
     }
 
-    private List<double> classValues;
-    public List<double> ClassValues {
+    private List<double> classValuesCache;
+    private List<double> ClassValuesCache {
       get {
-        if (classValues == null) {
-          classValues = Dataset.GetDoubleValues(TargetVariableParameter.Value.Value).Distinct().ToList();
-          classValues.Sort();
+        if (classValuesCache == null) {
+          classValuesCache = Dataset.GetDoubleValues(TargetVariableParameter.Value.Value).Distinct().OrderBy(x => x).ToList();
         }
-        return classValues;
+        return classValuesCache;
       }
     }
-    IEnumerable<double> IClassificationProblemData.ClassValues {
-      get { return ClassValues; }
-    }
-
+    public IEnumerable<double> ClassValues {
+      get { return ClassValuesCache; }
+    }
     public int Classes {
-      get { return ClassValues.Count; }
-    }
-
-    private List<string> classNames;
-    public List<string> ClassNames {
+      get { return ClassValuesCache.Count; }
+    }
+
+    private List<string> classNamesCache;
+    private List<string> ClassNamesCache {
       get {
-        if (classNames == null) {
-          classNames = new List<string>();
+        if (classNamesCache == null) {
+          classNamesCache = new List<string>();
           for (int i = 0; i < ClassNamesParameter.Value.Rows; i++)
-            classNames.Add(ClassNamesParameter.Value[i, 0]);
+            classNamesCache.Add(ClassNamesParameter.Value[i, 0]);
         }
-        return classNames;
+        return classNamesCache;
       }
     }
-    IEnumerable<string> IClassificationProblemData.ClassNames {
-      get { return ClassNames; }
-    }
-
-    private Dictionary<Tuple<double, double>, double> classificationPenaltiesCache = new Dictionary<Tuple<double, double>, double>();
+    public IEnumerable<string> ClassNames {
+      get { return ClassNamesCache; }
+    }
     #endregion
 
@@ -319 +315 @@
       DeregisterParameterEvents();
 
-      classNames = null;
       ((IStringConvertibleMatrix)ClassNamesParameter.Value).Columns = 1;
-      ((IStringConvertibleMatrix)ClassNamesParameter.Value).Rows = ClassValues.Count;
+      ((IStringConvertibleMatrix)ClassNamesParameter.Value).Rows = ClassValuesCache.Count;
       for (int i = 0; i < Classes; i++)
-        ClassNamesParameter.Value[i, 0] = "Class " + ClassValues[i];
+        ClassNamesParameter.Value[i, 0] = "Class " + ClassValuesCache[i];
       ClassNamesParameter.Value.ColumnNames = new List<string>() { "ClassNames" };
       ClassNamesParameter.Value.RowNames = ClassValues.Select(s => "ClassValue: " + s);
 
-      classificationPenaltiesCache.Clear();
-      ((ValueParameter<DoubleMatrix>)ClassificationPenaltiesParameter).ReactOnValueToStringChangedAndValueItemImageChanged = false;
       ((IStringConvertibleMatrix)ClassificationPenaltiesParameter.Value).Rows = Classes;
       ((IStringConvertibleMatrix)ClassificationPenaltiesParameter.Value).Columns = Classes;
@@ -339 +332 @@
         }
       }
-      ((ValueParameter<DoubleMatrix>)ClassificationPenaltiesParameter).ReactOnValueToStringChangedAndValueItemImageChanged = true;
       RegisterParameterEvents();
     }
 
     public string GetClassName(double classValue) {
-      if (!ClassValues.Contains(classValue)) throw new ArgumentException();
-      int index = ClassValues.IndexOf(classValue);
-      return ClassNames[index];
+      if (!ClassValuesCache.Contains(classValue)) throw new ArgumentException();
+      int index = ClassValuesCache.IndexOf(classValue);
+      return ClassNamesCache[index];
     }
     public double GetClassValue(string className) {
-      if (!ClassNames.Contains(className)) throw new ArgumentException();
-      int index = ClassNames.IndexOf(className);
-      return ClassValues[index];
+      if (!ClassNamesCache.Contains(className)) throw new ArgumentException();
+      int index = ClassNamesCache.IndexOf(className);
+      return ClassValuesCache[index];
     }
     public void SetClassName(double classValue, string className) {
-      if (!classValues.Contains(classValue)) throw new ArgumentException();
-      int index = ClassValues.IndexOf(classValue);
-      ClassNames[index] = className;
+      if (!ClassValuesCache.Contains(classValue)) throw new ArgumentException();
+      int index = ClassValuesCache.IndexOf(classValue);
       ClassNamesParameter.Value[index, 0] = className;
+      // updating of class names cache is not necessary here as the parameter value fires a changed event which updates the cache
     }
 
@@ -364 +356 @@
     }
     public double GetClassificationPenalty(double correctClassValue, double estimatedClassValue) {
-      var key = Tuple.Create(correctClassValue, estimatedClassValue);
-      if (!classificationPenaltiesCache.ContainsKey(key)) {
-        int correctClassIndex = ClassValues.IndexOf(correctClassValue);
-        int estimatedClassIndex = ClassValues.IndexOf(estimatedClassValue);
-        classificationPenaltiesCache[key] = ClassificationPenaltiesParameter.Value[correctClassIndex, estimatedClassIndex];
-      }
-      return classificationPenaltiesCache[key];
+      int correctClassIndex = ClassValuesCache.IndexOf(correctClassValue);
+      int estimatedClassIndex = ClassValuesCache.IndexOf(estimatedClassValue);
+      return ClassificationPenaltiesParameter.Value[correctClassIndex, estimatedClassIndex];
     }
     public void SetClassificationPenalty(string correctClassName, string estimatedClassName, double penalty) {
@@ -376 +364 @@
     }
     public void SetClassificationPenalty(double correctClassValue, double estimatedClassValue, double penalty) {
-      var key = Tuple.Create(correctClassValue, estimatedClassValue);
-      int correctClassIndex = ClassValues.IndexOf(correctClassValue);
-      int estimatedClassIndex = ClassValues.IndexOf(estimatedClassValue);
+      int correctClassIndex = ClassValuesCache.IndexOf(correctClassValue);
+      int estimatedClassIndex = ClassValuesCache.IndexOf(estimatedClassValue);
 
       ClassificationPenaltiesParameter.Value[correctClassIndex, estimatedClassIndex] = penalty;
@@ -388 +375 @@
       ClassNamesParameter.Value.Reset += new EventHandler(Parameter_ValueChanged);
       ClassNamesParameter.Value.ItemChanged += new EventHandler<EventArgs<int, int>>(MatrixParameter_ItemChanged);
-      ClassificationPenaltiesParameter.Value.Reset += new EventHandler(Parameter_ValueChanged);
-      ClassificationPenaltiesParameter.Value.ItemChanged += new EventHandler<EventArgs<int, int>>(MatrixParameter_ItemChanged);
     }
     private void DeregisterParameterEvents() {
@@ -395 +380 @@
       ClassNamesParameter.Value.Reset -= new EventHandler(Parameter_ValueChanged);
       ClassNamesParameter.Value.ItemChanged -= new EventHandler<EventArgs<int, int>>(MatrixParameter_ItemChanged);
-      ClassificationPenaltiesParameter.Value.Reset -= new EventHandler(Parameter_ValueChanged);
-      ClassificationPenaltiesParameter.Value.ItemChanged -= new EventHandler<EventArgs<int, int>>(MatrixParameter_ItemChanged);
     }
 
     private void TargetVariableParameter_ValueChanged(object sender, EventArgs e) {
-      classValues = null;
+      classValuesCache = null;
+      classNamesCache = null;
       ResetTargetVariableDependentMembers();
       OnChanged();
     }
     private void Parameter_ValueChanged(object sender, EventArgs e) {
+      classNamesCache = null;
       OnChanged();
     }
     private void MatrixParameter_ItemChanged(object sender, EventArgs<int, int> e) {
+      classNamesCache = null;
       OnChanged();
     }

trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ThresholdCalculators/AccuracyMaximizationThresholdCalculator.cs

@@ -85 +85 @@
             //all positives
             if (pair.TargetClassValue.IsAlmost(classValues[i - 1])) {
-              if (pair.EstimatedValue > lowerThreshold && pair.EstimatedValue < actualThreshold)
+              if (pair.EstimatedValue > lowerThreshold && pair.EstimatedValue <= actualThreshold)
                 //true positive
-                classificationScore += problemData.GetClassificationPenalty(classValues[i - 1], classValues[i - 1]);
+                classificationScore += problemData.GetClassificationPenalty(pair.TargetClassValue, pair.TargetClassValue);
               else
                 //false negative
-                classificationScore += problemData.GetClassificationPenalty(classValues[i], classValues[i - 1]);
+                classificationScore += problemData.GetClassificationPenalty(pair.TargetClassValue, classValues[i]);
             }
               //all negatives
             else {
-              if (pair.EstimatedValue > lowerThreshold && pair.EstimatedValue < actualThreshold)
+              if (pair.EstimatedValue > lowerThreshold && pair.EstimatedValue <= actualThreshold)
                 //false positive
-                classificationScore += problemData.GetClassificationPenalty(classValues[i - 1], classValues[i]);
+                classificationScore += problemData.GetClassificationPenalty(pair.TargetClassValue, classValues[i - 1]);
               else
                 //true negative, consider only upper class
-                classificationScore += problemData.GetClassificationPenalty(classValues[i], classValues[i]);
+                classificationScore += problemData.GetClassificationPenalty(pair.TargetClassValue, pair.TargetClassValue);
             }
           }

trunk/sources/HeuristicLab.Tests/HeuristicLab-3.3/SamplesTest.cs

@@ -343 +343 @@
       Assert.AreEqual(100.62175156249987, GetDoubleResult(ga, "CurrentWorstQuality"), 1E-8);
       Assert.AreEqual(100900, GetIntResult(ga, "EvaluatedSolutions"));
+      var bestTrainingSolution = (IClassificationSolution)ga.Results["Best training solution"].Value;
+      Assert.AreEqual(0.80625, bestTrainingSolution.TrainingAccuracy, 1E-8);
+      Assert.AreEqual(0.782608695652174, bestTrainingSolution.TestAccuracy, 1E-8);
     }