Changeset 7562

Timestamp:

03/06/12 15:08:13 (13 years ago)

Author:

sforsten

Message:

#1776:

• bug fix in NeighbourhoodWeightCalculator
• added GetConfidence method to IClassificationEnsembleSolutionWeightCalculator
• adjusted the confidence column in ClassificationEnsembleSolutionEstimatedClassValuesView

Location:

branches/ClassificationEnsembleVoting

Files:

• HeuristicLab.Problems.DataAnalysis.Views/3.4/Classification/ClassificationEnsembleSolutionEstimatedClassValuesView.cs (modified) (4 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationEnsembleSolution.cs (modified) (9 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/ClassificationWeightCalculator.cs (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/DiscriminantClassificationWeightCalculator.cs (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/MedianThresholdCalculator.cs (modified) (3 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/NeighbourhoodWeightCalculator.cs (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis/3.4/Interfaces/Classification/IClassificationEnsembleSolutionWeightCalculator.cs (modified) (1 diff)

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis.Views/3.4/Classification/ClassificationEnsembleSolutionEstimatedClassValuesView.cs

@@ -28 +28 @@
 using HeuristicLab.MainForm;
 using HeuristicLab.MainForm.WindowsForms;
+using HeuristicLab.Problems.DataAnalysis.Interfaces.Classification;
 
 namespace HeuristicLab.Problems.DataAnalysis.Views {
@@ -96 +97 @@
       }
 
+      IEnumerable<IClassificationSolution> solutions = Content.ClassificationSolutions.CheckedItems;
       int classValuesCount = Content.ProblemData.ClassValues.Count;
-      int solutionsCount = Content.ClassificationSolutions.Count();
+      int solutionsCount = solutions.Count();
       string[,] values = new string[indizes.Length, 5 + classValuesCount + solutionsCount];
       double[] target = Content.ProblemData.Dataset.GetDoubleValues(Content.ProblemData.TargetVariable).ToArray();
       List<List<double?>> estimatedValuesVector = GetEstimatedValues(SamplesComboBox.SelectedItem.ToString(), indizes,
-                                                            Content.ClassificationSolutions);
-      //List<double> weights = Content.Weights.ToList();
-      //double weightSum = weights.Sum();
+                                                            solutions);
+
+      IClassificationEnsembleSolutionWeightCalculator weightCalc = Content.WeightCalculator;
 
       for (int i = 0; i < indizes.Length; i++) {
@@ -113 +115 @@
           values[i, 2] = estimatedClassValues[i].ToString();
           values[i, 3] = (target[i].IsAlmost(estimatedClassValues[i])).ToString();
-
-          //currently disabled for test purpose
-
-          //IEnumerable<int> indices = FindAllIndices(estimatedValuesVector[i], estimatedClassValues[i]);
-          //double confidence = 0.0;
-          //foreach (var index in indices) {
-          //  confidence += weights[index];
-          //}
-          //values[i, 4] = (confidence / weightSum).ToString();
-          //var estimationCount = groups.Where(g => g.Key != null).Select(g => g.Count).Sum();
-          //values[i, 4] =
-          //  (((double)groups.Where(g => g.Key == estimatedClassValues[i]).Single().Count) / estimationCount).ToString();
-          values[i, 4] = "1.0";
+          values[i, 4] = weightCalc.GetConfidence(solutions, indizes[i], estimatedClassValues[i]).ToString();
 
           var groups =
@@ -145 +135 @@
       List<string> columnNames = new List<string>() { "Id", TargetClassValuesColumnName, EstimatedClassValuesColumnName, CorrectClassificationColumnName, ConfidenceColumnName };
       columnNames.AddRange(Content.ProblemData.ClassNames);
-      columnNames.AddRange(Content.Model.Models.Select(m => m.Name));
+      columnNames.AddRange(Content.ClassificationSolutions.CheckedItems.Select(s => s.Model.Name));//.Model.Models.Select(m => m.Name));
       matrix.ColumnNames = columnNames;
       matrix.SortableView = true;

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationEnsembleSolution.cs

@@ -51 +51 @@
     }
 
-    //[Storable]
-    //private Dictionary<IClassificationModel, IntRange> trainingPartitions;
-    //[Storable]
-    //private Dictionary<IClassificationModel, IntRange> testPartitions;
-
     private IClassificationEnsembleSolutionWeightCalculator weightCalculator;
 
@@ -66 +61 @@
         }
       }
+      get { return weightCalculator; }
     }
 
@@ -77 +73 @@
       foreach (var model in Model.Models) {
         IClassificationProblemData problemData = (IClassificationProblemData)ProblemData.Clone();
-        //problemData.TrainingPartition.Start = trainingPartitions[model].Start;
-        //problemData.TrainingPartition.End = trainingPartitions[model].End;
-        //problemData.TestPartition.Start = testPartitions[model].Start;
-        //problemData.TestPartition.End = testPartitions[model].End;
-
         classificationSolutions.Add(model.CreateClassificationSolution(problemData));
       }
@@ -89 +80 @@
     private ClassificationEnsembleSolution(ClassificationEnsembleSolution original, Cloner cloner)
       : base(original, cloner) {
-      //trainingPartitions = new Dictionary<IClassificationModel, IntRange>();
-      //testPartitions = new Dictionary<IClassificationModel, IntRange>();
-      //foreach (var pair in original.trainingPartitions) {
-      //  trainingPartitions[cloner.Clone(pair.Key)] = cloner.Clone(pair.Value);
-      //}
-      //foreach (var pair in original.testPartitions) {
-      //  testPartitions[cloner.Clone(pair.Key)] = cloner.Clone(pair.Value);
-      //}
-
       classificationSolutions = cloner.Clone(original.classificationSolutions);
       RegisterClassificationSolutionsEventHandler();
@@ -104 +86 @@
     public ClassificationEnsembleSolution()
       : base(new ClassificationEnsembleModel(), ClassificationEnsembleProblemData.EmptyProblemData) {
-      //trainingPartitions = new Dictionary<IClassificationModel, IntRange>();
-      //testPartitions = new Dictionary<IClassificationModel, IntRange>();
       classificationSolutions = new CheckedItemCollection<IClassificationSolution>();
 
@@ -121 +101 @@
     public ClassificationEnsembleSolution(IEnumerable<IClassificationModel> models, IClassificationProblemData problemData, IEnumerable<IntRange> trainingPartitions, IEnumerable<IntRange> testPartitions)
       : base(new ClassificationEnsembleModel(Enumerable.Empty<IClassificationModel>()), new ClassificationEnsembleProblemData(problemData)) {
-      //this.trainingPartitions = new Dictionary<IClassificationModel, IntRange>();
-      //this.testPartitions = new Dictionary<IClassificationModel, IntRange>();
       this.classificationSolutions = new CheckedItemCollection<IClassificationSolution>();
 
@@ -217 +195 @@
           solution.ProblemData = problemData;
       }
-      //foreach (var trainingPartition in trainingPartitions.Values) {
-      //  trainingPartition.Start = ProblemData.TrainingPartition.Start;
-      //  trainingPartition.End = ProblemData.TrainingPartition.End;
-      //}
-      //foreach (var testPartition in testPartitions.Values) {
-      //  testPartition.Start = ProblemData.TestPartition.Start;
-      //  testPartition.End = ProblemData.TestPartition.End;
-      //}
-
       base.OnProblemDataChanged();
     }
@@ -256 +225 @@
       if (Model.Models.Contains(solution.Model)) throw new ArgumentException();
       Model.Add(solution.Model);
-      //trainingPartitions[solution.Model] = solution.ProblemData.TrainingPartition;
-      //testPartitions[solution.Model] = solution.ProblemData.TestPartition;
     }
 
@@ -263 +230 @@
       if (!Model.Models.Contains(solution.Model)) throw new ArgumentException();
       Model.Remove(solution.Model);
-      //trainingPartitions.Remove(solution.Model);
-      //testPartitions.Remove(solution.Model);
     }
   }

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/ClassificationWeightCalculator.cs

@@ -113 +113 @@
     }
 
+    public virtual double GetConfidence(IEnumerable<IClassificationSolution> solutions, int index, double estimatedClassValue) {
+      if (solutions.Count() < 1)
+        return double.NaN;
+      Dataset dataset = solutions.First().ProblemData.Dataset;
+      var correctSolutions = solutions.Select(s => new { Solution = s, Values = s.Model.GetEstimatedClassValues(dataset, Enumerable.Repeat(index, 1)).First() })
+                                      .Where(a => a.Values.Equals(estimatedClassValue))
+                                      .Select(a => a.Solution);
+      return (from sol in correctSolutions
+              select weights[sol]).Sum();
+    }
+
     #region Helper
     protected IEnumerable<double> GetValues(IList<double> targetValues, IEnumerable<int> indizes) {

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/DiscriminantClassificationWeightCalculator.cs

@@ -82 +82 @@
       }
     }
+
+    public sealed override double GetConfidence(IEnumerable<IClassificationSolution> solutions, int index, double estimatedClassValue) {
+      if (solutions.Count() < 1 || !solutions.All(x => x is IDiscriminantFunctionClassificationSolution))
+        return double.NaN;
+
+      IEnumerable<IDiscriminantFunctionClassificationSolution> discriminantSolutions = solutions.Cast<IDiscriminantFunctionClassificationSolution>();
+
+      return GetDiscriminantConfidence(discriminantSolutions, index, estimatedClassValue);
+    }
+
+    protected virtual double GetDiscriminantConfidence(IEnumerable<IDiscriminantFunctionClassificationSolution> solutions, int index, double estimatedClassValue) {
+      return base.GetConfidence(solutions, index, estimatedClassValue);
+    }
   }
 }

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/MedianThresholdCalculator.cs

@@ -20 +20 @@
 #endregion
 
+using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
@@ -84 +85 @@
       }
       AccuracyMaximizationThresholdCalculator.CalculateThresholds(solutionProblemData[0], median, targetValues, out classValues, out threshold);
-      return median;
+      return Enumerable.Repeat<double>(1, discriminantSolutions.Count());
     }
 
     protected override double DiscriminantAggregateEstimatedClassValues(IDictionary<IClassificationSolution, double> estimatedClassValues, IDictionary<IClassificationSolution, double> estimatedValues) {
-      double classValue = classValues.First();
       IList<double> values = estimatedValues.Select(x => x.Value).ToList();
       if (values.Count <= 0)
         return double.NaN;
       double median = GetMedian(values);
+      return GetClassValueToMedian(median);
+    }
+    private double GetClassValueToMedian(double median) {
+      double classValue = classValues.First();
       for (int i = 0; i < classValues.Count(); i++) {
         if (median > threshold[i])
@@ -100 +104 @@
       }
       return classValue;
+    }
+
+    protected override double GetDiscriminantConfidence(IEnumerable<IDiscriminantFunctionClassificationSolution> solutions, int index, double estimatedClassValue) {
+      // only works with binary classification
+      if (!classValues.Count().Equals(2))
+        return double.NaN;
+      Dataset dataset = solutions.First().ProblemData.Dataset;
+      IList<double> values = solutions.Select(s => s.Model.GetEstimatedValues(dataset, Enumerable.Repeat(index, 1)).First()).ToList();
+      if (values.Count <= 0)
+        return double.NaN;
+      double median = GetMedian(values);
+      if (estimatedClassValue.Equals(classValues[0])) {
+        if (median < estimatedClassValue)
+          return 1;
+        else if (median >= threshold[1])
+          return 0;
+        else {
+          double distance = threshold[1] - classValues[0];
+          return (1 / distance) * (median - classValues[0]);
+        }
+      } else if (estimatedClassValue.Equals(classValues[1])) {
+        if (median > estimatedClassValue)
+          return 1;
+        else if (median <= threshold[1])
+          return 0;
+        else {
+          double distance = classValues[1] - threshold[1];
+          return (1 / distance) * (classValues[1] - median);
+        }
+      } else
+        return double.NaN;
     }
 

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/NeighbourhoodWeightCalculator.cs

@@ -58 +58 @@
       foreach (var solution in discriminantSolutions) {
         estimatedValues.Add(solution.Model.GetEstimatedValues(dataSet, rows).ToList());
-        estimatedClassValues.Add(solution.Model.GetEstimatedValues(dataSet, rows).ToList());
+        estimatedClassValues.Add(solution.Model.GetEstimatedClassValues(dataSet, rows).ToList());
       }
 

branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Interfaces/Classification/IClassificationEnsembleSolutionWeightCalculator.cs

@@ -25 +25 @@
 namespace HeuristicLab.Problems.DataAnalysis.Interfaces.Classification {
   public delegate bool CheckPoint(IClassificationProblemData problemData, int point);
+
   public interface IClassificationEnsembleSolutionWeightCalculator : INamedItem {
     void CalculateNormalizedWeights(IEnumerable<IClassificationSolution> classificationSolutions);
     IEnumerable<double> AggregateEstimatedClassValues(IEnumerable<IClassificationSolution> solutions, Dataset dataset, IEnumerable<int> rows, CheckPoint handler);
+    double GetConfidence(IEnumerable<IClassificationSolution> solutions, int index, double estimatedClassValue);
+
     CheckPoint GetTestClassDelegate();
     CheckPoint GetTrainingClassDelegate();