Changeset 8297 for branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis

Timestamp:

07/17/12 15:30:04 (12 years ago)

Author:

sforsten

Message:

#1776:

• Corrected namespace of IClassificationEnsembleSolutionWeightCalculator interface
• Corrected calculation of confidence for test and training samples in ClassificationEnsembleSolutionEstimatedClassValuesView
• Added overload method GetConfidence to IClassificationEnsembleSolutionWeightCalculator to calculate more than one point at a time (maybe additional methods for training and test confidence could improve the performance remarkably)
• Added ClassificationEnsembleSolutionConfidenceAccuracyDependence to see how accuracy would behave, if only samples with high confidence would be classified

Location:

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4

Files:

• Implementation/Classification/ClassificationEnsembleSolution.cs (modified) (1 diff)
• Implementation/Classification/WeightCalculators/AverageThresholdCalculator.cs (modified) (1 diff)
• Implementation/Classification/WeightCalculators/ClassificationWeightCalculator.cs (modified) (2 diffs)
• Implementation/Classification/WeightCalculators/DiscriminantClassificationWeightCalculator.cs (modified) (2 diffs)
• Implementation/Classification/WeightCalculators/MajorityVoteWeightCalculator.cs (modified) (1 diff)
• Implementation/Classification/WeightCalculators/MedianThresholdCalculator.cs (modified) (1 diff)
• Interfaces/Classification/IClassificationEnsembleSolutionWeightCalculator.cs (modified) (2 diffs)

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationEnsembleSolution.cs

@@ -28 +28 @@
 using HeuristicLab.Data;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using HeuristicLab.Problems.DataAnalysis.Interfaces.Classification;
+using HeuristicLab.Problems.DataAnalysis.Interfaces;
 
 namespace HeuristicLab.Problems.DataAnalysis {

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/AverageThresholdCalculator.cs

@@ -66 +66 @@
         return double.NaN;
       double avg = values.Average();
+      return GetAverageConfidence(avg, estimatedClassValue);
+    }
+
+    public override IEnumerable<double> GetDiscriminantConfidence(IEnumerable<IDiscriminantFunctionClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue) {
+      if (!classValues.Count().Equals(2))
+        return Enumerable.Repeat(double.NaN, indices.Count());
+
+      Dataset dataset = solutions.First().ProblemData.Dataset;
+      double[][] values = solutions.Select(s => s.Model.GetEstimatedValues(dataset, indices).ToArray()).ToArray();
+      double[] confidences = new double[indices.Count()];
+      double[] estimatedClassValueArr = estimatedClassValue.ToArray();
+
+      for (int i = 0; i < indices.Count(); i++) {
+        double avg = values.Select(x => x[i]).Average();
+        confidences[i] = GetAverageConfidence(avg, estimatedClassValueArr[i]);
+      }
+
+      return confidences;
+    }
+
+    protected double GetAverageConfidence(double avg, double estimatedClassValue) {
       if (estimatedClassValue.Equals(classValues[0])) {
         if (avg < estimatedClassValue)

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/ClassificationWeightCalculator.cs

@@ -26 +26 @@
 using HeuristicLab.Data;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using HeuristicLab.Problems.DataAnalysis.Interfaces.Classification;
+using HeuristicLab.Problems.DataAnalysis.Interfaces;
 
 namespace HeuristicLab.Problems.DataAnalysis {
@@ -124 +124 @@
     }
 
+    public virtual IEnumerable<double> GetConfidence(IEnumerable<IClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue) {
+      if (solutions.Count() < 1)
+        return Enumerable.Repeat(double.NaN, indices.Count());
+
+      Dataset dataset = solutions.First().ProblemData.Dataset;
+      Dictionary<IClassificationSolution, double[]> solValues = solutions.ToDictionary(x => x, x => x.Model.GetEstimatedClassValues(dataset, indices).ToArray());
+      double[] estimatedClassValueArr = estimatedClassValue.ToArray();
+      double[] confidences = new double[indices.Count()];
+
+      for (int i = 0; i < indices.Count(); i++) {
+        var correctSolutions = solValues.Where(x => DoubleExtensions.IsAlmost(x.Value[i], estimatedClassValueArr[i]));
+        confidences[i] = (from sol in correctSolutions
+                          select weights[sol.Key]).Sum();
+      }
+
+      return confidences;
+    }
+
     #region Helper
     protected IEnumerable<double> GetValues(IList<double> targetValues, IEnumerable<int> indizes) {

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/DiscriminantClassificationWeightCalculator.cs

@@ -24 +24 @@
 using HeuristicLab.Common;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using HeuristicLab.Problems.DataAnalysis.Interfaces.Classification;
+using HeuristicLab.Problems.DataAnalysis.Interfaces;
 
 namespace HeuristicLab.Problems.DataAnalysis {
@@ -95 +95 @@
       return base.GetConfidence(solutions, index, estimatedClassValue);
     }
+
+    public sealed override IEnumerable<double> GetConfidence(IEnumerable<IClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue) {
+      if (solutions.Count() < 1 || !solutions.All(x => x is IDiscriminantFunctionClassificationSolution))
+        return Enumerable.Repeat(double.NaN, indices.Count());
+
+      IEnumerable<IDiscriminantFunctionClassificationSolution> discriminantSolutions = solutions.Cast<IDiscriminantFunctionClassificationSolution>();
+
+      return GetDiscriminantConfidence(discriminantSolutions, indices, estimatedClassValue);
+    }
+
+    public virtual IEnumerable<double> GetDiscriminantConfidence(IEnumerable<IDiscriminantFunctionClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue) {
+      return base.GetConfidence(solutions, indices, estimatedClassValue);
+    }
   }
 }

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/MajorityVoteWeightCalculator.cs

@@ -61 +61 @@
       return ((double)correctEstimated / (double)solutions.Count() - 0.5) * 2;
     }
+
+    public override IEnumerable<double> GetConfidence(IEnumerable<IClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue) {
+      if (solutions.Count() < 1)
+        return Enumerable.Repeat(double.NaN, indices.Count());
+      Dataset dataset = solutions.First().ProblemData.Dataset;
+      var estimationsPerSolution = solutions.Select(s => s.Model.GetEstimatedClassValues(dataset, indices).ToArray()).ToArray();
+      double[] estimatedClassValueArr = estimatedClassValue.ToArray();
+      double correctEstimated;
+      double[] confidences = new double[indices.Count()];
+
+      for (int i = 0; i < indices.Count(); i++) {
+        correctEstimated = estimationsPerSolution.Where(x => DoubleExtensions.IsAlmost(x[i], estimatedClassValueArr[i])).Count();
+        confidences[i] = (correctEstimated / (double)solutions.Count() - 0.5) * 2;
+      }
+
+      return confidences;
+    }
   }
 }

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/MedianThresholdCalculator.cs

@@ -66 +66 @@
         return double.NaN;
       double median = GetMedian(values);
+      return GetMedianConfidence(median, estimatedClassValue);
+    }
+
+    public override IEnumerable<double> GetDiscriminantConfidence(IEnumerable<IDiscriminantFunctionClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue) {
+      if (!classValues.Count().Equals(2))
+        return Enumerable.Repeat(double.NaN, indices.Count());
+
+      Dataset dataset = solutions.First().ProblemData.Dataset;
+      double[][] values = solutions.Select(s => s.Model.GetEstimatedValues(dataset, indices).ToArray()).ToArray();
+      double[] confidences = new double[indices.Count()];
+      double[] estimatedClassValueArr = estimatedClassValue.ToArray();
+
+      for (int i = 0; i < indices.Count(); i++) {
+        double avg = values.Select(x => x[i]).Average();
+        confidences[i] = GetMedianConfidence(avg, estimatedClassValueArr[i]);
+      }
+
+      return confidences;
+    }
+
+    protected double GetMedianConfidence(double median, double estimatedClassValue) {
       if (estimatedClassValue.Equals(classValues[0])) {
         if (median < estimatedClassValue)

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Interfaces/Classification/IClassificationEnsembleSolutionWeightCalculator.cs

@@ -23 +23 @@
 using HeuristicLab.Core;
 
-namespace HeuristicLab.Problems.DataAnalysis.Interfaces.Classification {
+namespace HeuristicLab.Problems.DataAnalysis.Interfaces {
   public delegate bool CheckPoint(IClassificationProblemData problemData, int point);
 
@@ -30 +30 @@
     IEnumerable<double> AggregateEstimatedClassValues(IEnumerable<IClassificationSolution> solutions, Dataset dataset, IEnumerable<int> rows, CheckPoint handler);
     double GetConfidence(IEnumerable<IClassificationSolution> solutions, int index, double estimatedClassValue);
+    IEnumerable<double> GetConfidence(IEnumerable<IClassificationSolution> solutions, IEnumerable<int> indices, IEnumerable<double> estimatedClassValue);
 
     CheckPoint GetTestClassDelegate();