Changeset 8814 for branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification

Timestamp:

10/16/12 15:19:40 (12 years ago)

Author:

sforsten

Message:

#1776:

• improved performance of confidence calculation
• fixed bug in median confidence calculation
• fixed bug in average confidence calculation
• confidence calculation is now easier for training and test
• removed obsolete view ClassificationEnsembleSolutionConfidenceAccuracyDependence

Location:

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators

Files:

• AverageThresholdCalculator.cs (modified) (3 diffs)
• ClassificationWeightCalculator.cs (modified) (4 diffs)
• DiscriminantClassificationWeightCalculator.cs (modified) (3 diffs)
• MajorityVoteWeightCalculator.cs (modified) (1 diff)
• MedianThresholdCalculator.cs (modified) (3 diffs)

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/AverageThresholdCalculator.cs

@@ -57 +57 @@
     }
 
-    protected override double GetDiscriminantConfidence(IEnumerable<IDiscriminantFunctionClassificationSolution> solutions, int index, double estimatedClassValue) {
+    protected override double GetDiscriminantConfidence(IEnumerable<IDiscriminantFunctionClassificationSolution> solutions, int index, double estimatedClassValue, CheckPoint handler) {
       Dataset dataset = solutions.First().ProblemData.Dataset;
-      IList<double> values = solutions.Select(s => s.Model.GetEstimatedValues(dataset, Enumerable.Repeat(index, 1)).First()).ToList();
+      IList<double> values = solutions.Where(s => handler(s.ProblemData, index)).Select(s => s.Model.GetEstimatedValues(dataset, Enumerable.Repeat(index, 1)).First()).ToList();
       if (values.Count <= 0)
         return double.NaN;
@@ -66 +66 @@
     }
 
-    public override IEnumerable<double> GetDiscriminantConfidence(IEnumerable<IDiscriminantFunctionClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue) {
+    public override IEnumerable<double> GetDiscriminantConfidence(IEnumerable<IDiscriminantFunctionClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue, CheckPoint handler) {
       Dataset dataset = solutions.First().ProblemData.Dataset;
-      double[][] values = solutions.Select(s => s.Model.GetEstimatedValues(dataset, indices).ToArray()).ToArray();
+      List<int> indicesList = indices.ToList();
+      var solValues = solutions.ToDictionary(x => x, x => x.Model.GetEstimatedValues(dataset, indicesList).ToArray());
       double[] confidences = new double[indices.Count()];
       double[] estimatedClassValueArr = estimatedClassValue.ToArray();
 
-      for (int i = 0; i < indices.Count(); i++) {
-        double avg = values.Select(x => x[i]).Average();
-        confidences[i] = GetAverageConfidence(avg, estimatedClassValueArr[i]);
+      for (int i = 0; i < indicesList.Count; i++) {
+        var values = solValues.Where(x => handler(x.Key.ProblemData, indicesList[i])).Select(x => x.Value[i]);
+        if (values.Count() <= 0) {
+          confidences[i] = double.NaN;
+        } else {
+          double avg = values.Average();
+          confidences[i] = GetAverageConfidence(avg, estimatedClassValueArr[i]);
+        }
       }
 
@@ -84 +90 @@
         if (estimatedClassValue.Equals(classValues[i])) {
           //special case: avgerage is higher than value of highest class
-          if (i == classValues.Length - 1 && avg > estimatedClassValue) {
+          if (i == classValues.Length - 1 && avg >= estimatedClassValue) {
             return 1;
           }

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/ClassificationWeightCalculator.cs

@@ -24 +24 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
-using HeuristicLab.Data;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
@@ -112 +111 @@
     }
 
-    public virtual double GetConfidence(IEnumerable<IClassificationSolution> solutions, int index, double estimatedClassValue) {
+    public virtual double GetConfidence(IEnumerable<IClassificationSolution> solutions, int index, double estimatedClassValue, CheckPoint handler) {
       if (solutions.Count() < 1)
         return double.NaN;
       Dataset dataset = solutions.First().ProblemData.Dataset;
       var correctSolutions = solutions.Select(s => new { Solution = s, Values = s.Model.GetEstimatedClassValues(dataset, Enumerable.Repeat(index, 1)).First() })
-                                      .Where(a => a.Values.Equals(estimatedClassValue))
+                                      .Where(a => handler(a.Solution.ProblemData, index) && a.Values.Equals(estimatedClassValue))
                                       .Select(a => a.Solution);
       return (from sol in correctSolutions
@@ -123 +122 @@
     }
 
-    public virtual IEnumerable<double> GetConfidence(IEnumerable<IClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue) {
+    public virtual IEnumerable<double> GetConfidence(IEnumerable<IClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue, CheckPoint handler) {
       if (solutions.Count() < 1)
         return Enumerable.Repeat(double.NaN, indices.Count());
 
+      List<int> indicesList = indices.ToList();
+
       Dataset dataset = solutions.First().ProblemData.Dataset;
-      Dictionary<IClassificationSolution, double[]> solValues = solutions.ToDictionary(x => x, x => x.Model.GetEstimatedClassValues(dataset, indices).ToArray());
+      Dictionary<IClassificationSolution, double[]> solValues = solutions.ToDictionary(x => x, x => x.Model.GetEstimatedClassValues(dataset, indicesList).ToArray());
       double[] estimatedClassValueArr = estimatedClassValue.ToArray();
-      double[] confidences = new double[indices.Count()];
+      double[] confidences = new double[indicesList.Count];
 
-      for (int i = 0; i < indices.Count(); i++) {
+      for (int i = 0; i < indicesList.Count; i++) {
         var correctSolutions = solValues.Where(x => DoubleExtensions.IsAlmost(x.Value[i], estimatedClassValueArr[i]));
         confidences[i] = (from sol in correctSolutions
+                          where handler(sol.Key.ProblemData, indicesList[i])
                           select weights[sol.Key]).Sum();
       }
@@ -142 +144 @@
 
     #region Helper
-    protected IEnumerable<double> GetValues(IList<double> targetValues, IEnumerable<int> indizes) {
-      return from i in indizes
-             select targetValues[i];
-    }
     protected bool PointInTraining(IClassificationProblemData problemData, int point) {
-      IntRange trainingPartition = problemData.TrainingPartition;
-      IntRange testPartition = problemData.TestPartition;
-      return (trainingPartition.Start <= point && point < trainingPartition.End)
-        && !(testPartition.Start <= point && point < testPartition.End);
+      return problemData.IsTrainingSample(point);
     }
     protected bool PointInTest(IClassificationProblemData problemData, int point) {
-      IntRange testPartition = problemData.TestPartition;
-      return testPartition.Start <= point && point < testPartition.End;
+      return problemData.IsTestSample(point);
     }
     protected bool AllPoints(IClassificationProblemData problemData, int point) {

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/DiscriminantClassificationWeightCalculator.cs

@@ -82 +82 @@
     }
 
-    public sealed override double GetConfidence(IEnumerable<IClassificationSolution> solutions, int index, double estimatedClassValue) {
+    public sealed override double GetConfidence(IEnumerable<IClassificationSolution> solutions, int index, double estimatedClassValue, CheckPoint handler) {
       if (solutions.Count() < 1 || !solutions.All(x => x is IDiscriminantFunctionClassificationSolution))
         return double.NaN;
@@ -88 +88 @@
       IEnumerable<IDiscriminantFunctionClassificationSolution> discriminantSolutions = solutions.Cast<IDiscriminantFunctionClassificationSolution>();
 
-      return GetDiscriminantConfidence(discriminantSolutions, index, estimatedClassValue);
+      return GetDiscriminantConfidence(discriminantSolutions, index, estimatedClassValue, handler);
     }
 
-    protected virtual double GetDiscriminantConfidence(IEnumerable<IDiscriminantFunctionClassificationSolution> solutions, int index, double estimatedClassValue) {
-      return base.GetConfidence(solutions, index, estimatedClassValue);
+    protected virtual double GetDiscriminantConfidence(IEnumerable<IDiscriminantFunctionClassificationSolution> solutions, int index, double estimatedClassValue, CheckPoint handler) {
+      return base.GetConfidence(solutions, index, estimatedClassValue, handler);
     }
 
-    public sealed override IEnumerable<double> GetConfidence(IEnumerable<IClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue) {
+    public sealed override IEnumerable<double> GetConfidence(IEnumerable<IClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue, CheckPoint handler) {
       if (solutions.Count() < 1 || !solutions.All(x => x is IDiscriminantFunctionClassificationSolution))
         return Enumerable.Repeat(double.NaN, indices.Count());
@@ -101 +101 @@
       IEnumerable<IDiscriminantFunctionClassificationSolution> discriminantSolutions = solutions.Cast<IDiscriminantFunctionClassificationSolution>();
 
-      return GetDiscriminantConfidence(discriminantSolutions, indices, estimatedClassValue);
+      return GetDiscriminantConfidence(discriminantSolutions, indices, estimatedClassValue, handler);
     }
 
-    public virtual IEnumerable<double> GetDiscriminantConfidence(IEnumerable<IDiscriminantFunctionClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue) {
-      return base.GetConfidence(solutions, indices, estimatedClassValue);
+    public virtual IEnumerable<double> GetDiscriminantConfidence(IEnumerable<IDiscriminantFunctionClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue, CheckPoint handler) {
+      return base.GetConfidence(solutions, indices, estimatedClassValue, handler);
     }
   }

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/MajorityVoteWeightCalculator.cs

@@ -52 +52 @@
     }
 
-    public override double GetConfidence(IEnumerable<IClassificationSolution> solutions, int index, double estimatedClassValue) {
+    public override double GetConfidence(IEnumerable<IClassificationSolution> solutions, int index, double estimatedClassValue, CheckPoint handler) {
       if (solutions.Count() < 1)
         return double.NaN;
-      Dataset dataset = solutions.First().ProblemData.Dataset;
-      int correctEstimated = solutions.Select(s => s.Model.GetEstimatedClassValues(dataset, Enumerable.Repeat(index, 1)).First())
+      var votingSolutions = solutions.Where(s => handler(s.ProblemData, index));
+      if (votingSolutions.Count() < 1)
+        return double.NaN;
+      Dataset dataset = votingSolutions.First().ProblemData.Dataset;
+      int correctEstimated = votingSolutions.Select(s => s.Model.GetEstimatedClassValues(dataset, Enumerable.Repeat(index, 1)).First())
                                       .Where(x => x.Equals(estimatedClassValue))
                                       .Count();
-      return ((double)correctEstimated / (double)solutions.Count() - 0.5) * 2;
+      return ((double)correctEstimated / (double)votingSolutions.Count() - 0.5) * 2;
     }
 
-    public override IEnumerable<double> GetConfidence(IEnumerable<IClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue) {
+    public override IEnumerable<double> GetConfidence(IEnumerable<IClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue, CheckPoint handler) {
       if (solutions.Count() < 1)
         return Enumerable.Repeat(double.NaN, indices.Count());
+
+      List<int> indicesList = indices.ToList();
       Dataset dataset = solutions.First().ProblemData.Dataset;
-      var estimationsPerSolution = solutions.Select(s => s.Model.GetEstimatedClassValues(dataset, indices).ToArray()).ToArray();
+      var solValues = solutions.ToDictionary(x => x, x => x.Model.GetEstimatedClassValues(dataset, indicesList).ToArray());
       double[] estimatedClassValueArr = estimatedClassValue.ToArray();
       double correctEstimated;
       double[] confidences = new double[indices.Count()];
 
-      for (int i = 0; i < indices.Count(); i++) {
-        correctEstimated = estimationsPerSolution.Where(x => DoubleExtensions.IsAlmost(x[i], estimatedClassValueArr[i])).Count();
-        confidences[i] = (correctEstimated / (double)solutions.Count() - 0.5) * 2;
+      for (int i = 0; i < indicesList.Count; i++) {
+        var votingSolutions = solValues.Where(x => handler(x.Key.ProblemData, indicesList[i]));
+        correctEstimated = votingSolutions.Where(x => DoubleExtensions.IsAlmost(x.Value[i], estimatedClassValueArr[i])).Count();
+        confidences[i] = (correctEstimated / (double)votingSolutions.Count() - 0.5) * 2;
       }
 

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/MedianThresholdCalculator.cs

@@ -57 +57 @@
     }
 
-    protected override double GetDiscriminantConfidence(IEnumerable<IDiscriminantFunctionClassificationSolution> solutions, int index, double estimatedClassValue) {
-
+    protected override double GetDiscriminantConfidence(IEnumerable<IDiscriminantFunctionClassificationSolution> solutions, int index, double estimatedClassValue, CheckPoint handler) {
       Dataset dataset = solutions.First().ProblemData.Dataset;
-      IList<double> values = solutions.Select(s => s.Model.GetEstimatedValues(dataset, Enumerable.Repeat(index, 1)).First()).ToList();
+      IList<double> values = solutions.Where(s => handler(s.ProblemData, index)).Select(s => s.Model.GetEstimatedValues(dataset, Enumerable.Repeat(index, 1)).First()).ToList();
       if (values.Count <= 0)
         return double.NaN;
@@ -67 +66 @@
     }
 
-    public override IEnumerable<double> GetDiscriminantConfidence(IEnumerable<IDiscriminantFunctionClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue) {
+    public override IEnumerable<double> GetDiscriminantConfidence(IEnumerable<IDiscriminantFunctionClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue, CheckPoint handler) {
 
       Dataset dataset = solutions.First().ProblemData.Dataset;
-      double[][] values = solutions.Select(s => s.Model.GetEstimatedValues(dataset, indices).ToArray()).ToArray();
+      List<int> indicesList = indices.ToList();
+      var solValues = solutions.ToDictionary(x => x, x => x.Model.GetEstimatedValues(dataset, indicesList).ToArray());
       double[] confidences = new double[indices.Count()];
       double[] estimatedClassValueArr = estimatedClassValue.ToArray();
 
-      for (int i = 0; i < indices.Count(); i++) {
-        double avg = values.Select(x => x[i]).Average();
-        confidences[i] = GetMedianConfidence(avg, estimatedClassValueArr[i]);
+      for (int i = 0; i < indicesList.Count; i++) {
+        var values = solValues.Where(x => handler(x.Key.ProblemData, indicesList[i])).Select(x => x.Value[i]).ToList();
+        if (values.Count <= 0) {
+          confidences[i] = double.NaN;
+        } else {
+          double median = GetMedian(values);
+          confidences[i] = GetMedianConfidence(median, estimatedClassValueArr[i]);
+        }
       }
 
@@ -85 +90 @@
       for (int i = 0; i < classValues.Length; i++) {
         if (estimatedClassValue.Equals(classValues[i])) {
-          //special case: avgerage is higher than value of highest class
-          if (i == classValues.Length - 1 && median > estimatedClassValue) {
+          //special case: median is higher than value of highest class
+          if (i == classValues.Length - 1 && median >= estimatedClassValue) {
             return 1;
           }
-          //special case: average is lower than value of lowest class
+          //special case: median is lower than value of lowest class
           if (i == 0 && median < estimatedClassValue) {
             return 1;
           }
-          //special case: average is not between threshold of estimated class value
+          //special case: median is not between threshold of estimated class value
           if ((i < classValues.Length - 1 && median >= threshold[i + 1]) || median <= threshold[i]) {
             return 0;
           }
 
-          double thresholdToClassDistance, thresholdToAverageValueDistance;
+          double thresholdToClassDistance, thresholdToMedianValueDistance;
           if (median >= classValues[i]) {
             thresholdToClassDistance = threshold[i + 1] - classValues[i];
-            thresholdToAverageValueDistance = threshold[i + 1] - median;
+            thresholdToMedianValueDistance = threshold[i + 1] - median;
           } else {
             thresholdToClassDistance = classValues[i] - threshold[i];
-            thresholdToAverageValueDistance = median - threshold[i];
+            thresholdToMedianValueDistance = median - threshold[i];
           }
-          return (1 / thresholdToClassDistance) * thresholdToAverageValueDistance;
+          return (1 / thresholdToClassDistance) * thresholdToMedianValueDistance;
         }
       }