Changeset 9363 for branches/OaaS/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification

Timestamp:

04/16/13 13:13:41 (12 years ago)

Author:

spimming

Message:

#1888:

• Merged revisions from trunk

Location:

branches/OaaS

Files:

• . (modified) (2 props)
• HeuristicLab.Problems.DataAnalysis (modified) (1 prop)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationEnsembleModel.cs (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationEnsembleSolution.cs (modified) (3 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblemData.cs (modified) (8 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationSolution.cs (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationSolutionBase.cs (modified) (2 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/DiscriminantFunctionClassificationModel.cs (modified) (6 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/DiscriminantFunctionClassificationSolution.cs (modified) (2 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/DiscriminantFunctionClassificationSolutionBase.cs (modified) (4 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ThresholdCalculators/AccuracyMaximizationThresholdCalculator.cs (modified) (3 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ThresholdCalculators/NormalDistributionCutPointsThresholdCalculator.cs (modified) (4 diffs)

branches/OaaS

@@ -21 +21 @@
 protoc.exe
 _ReSharper.HeuristicLab 3.3 Tests
+Google.ProtocolBuffers-2.4.1.473.dll
 packages

@@ -21 +21 @@
 protoc.exe
 _ReSharper.HeuristicLab 3.3 Tests
+Google.ProtocolBuffers-2.4.1.473.dll
 packages

branches/OaaS/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationEnsembleModel.cs

@@ -95 +95 @@
 
     IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {
-      return new ClassificationEnsembleSolution(models, problemData);
+      return new ClassificationEnsembleSolution(models, new ClassificationEnsembleProblemData(problemData));
     }
     #endregion

branches/OaaS/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationEnsembleSolution.cs

@@ -36 +36 @@
   [Item("Classification Ensemble Solution", "A classification solution that contains an ensemble of multiple classification models")]
   [Creatable("Data Analysis - Ensembles")]
-  public sealed class ClassificationEnsembleSolution : ClassificationSolution, IClassificationEnsembleSolution {
+  public sealed class ClassificationEnsembleSolution : ClassificationSolutionBase, IClassificationEnsembleSolution {
     private readonly Dictionary<int, double> trainingEvaluationCache = new Dictionary<int, double>();
     private readonly Dictionary<int, double> testEvaluationCache = new Dictionary<int, double>();
+    private readonly Dictionary<int, double> evaluationCache = new Dictionary<int, double>();
 
     public new IClassificationEnsembleModel Model {
@@ -104 +105 @@
     }
 
+    public ClassificationEnsembleSolution(IClassificationProblemData problemData) :
+      this(Enumerable.Empty<IClassificationModel>(), problemData) { }
+
     public ClassificationEnsembleSolution(IEnumerable<IClassificationModel> models, IClassificationProblemData problemData)
       : this(models, problemData,
@@ -150 +154 @@
     }
 
-    protected override void RecalculateResults() {
-      CalculateResults();
-    }
 
     #region Evaluation
+    public override IEnumerable<double> EstimatedClassValues {
+      get { return GetEstimatedClassValues(Enumerable.Range(0, ProblemData.Dataset.Rows)); }
+    }
+
     public override IEnumerable<double> EstimatedTrainingClassValues {
       get {

branches/OaaS/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblemData.cs

@@ -223 +223 @@
     }
 
-    private List<double> classValues;
-    public List<double> ClassValues {
+    private List<double> classValuesCache;
+    private List<double> ClassValuesCache {
       get {
-        if (classValues == null) {
-          classValues = Dataset.GetDoubleValues(TargetVariableParameter.Value.Value).Distinct().ToList();
-          classValues.Sort();
+        if (classValuesCache == null) {
+          classValuesCache = Dataset.GetDoubleValues(TargetVariableParameter.Value.Value).Distinct().OrderBy(x => x).ToList();
         }
-        return classValues;
+        return classValuesCache;
       }
     }
-    IEnumerable<double> IClassificationProblemData.ClassValues {
-      get { return ClassValues; }
-    }
-
+    public IEnumerable<double> ClassValues {
+      get { return ClassValuesCache; }
+    }
     public int Classes {
-      get { return ClassValues.Count; }
-    }
-
-    private List<string> classNames;
-    public List<string> ClassNames {
+      get { return ClassValuesCache.Count; }
+    }
+
+    private List<string> classNamesCache;
+    private List<string> ClassNamesCache {
       get {
-        if (classNames == null) {
-          classNames = new List<string>();
+        if (classNamesCache == null) {
+          classNamesCache = new List<string>();
           for (int i = 0; i < ClassNamesParameter.Value.Rows; i++)
-            classNames.Add(ClassNamesParameter.Value[i, 0]);
+            classNamesCache.Add(ClassNamesParameter.Value[i, 0]);
         }
-        return classNames;
+        return classNamesCache;
       }
     }
-    IEnumerable<string> IClassificationProblemData.ClassNames {
-      get { return ClassNames; }
-    }
-
-    private Dictionary<Tuple<double, double>, double> classificationPenaltiesCache = new Dictionary<Tuple<double, double>, double>();
+    public IEnumerable<string> ClassNames {
+      get { return ClassNamesCache; }
+    }
     #endregion
 
@@ -277 +273 @@
 
     public ClassificationProblemData() : this(defaultDataset, defaultAllowedInputVariables, defaultTargetVariable) { }
+
+    public ClassificationProblemData(IClassificationProblemData classificationProblemData)
+      : this(classificationProblemData.Dataset, classificationProblemData.AllowedInputVariables, classificationProblemData.TargetVariable) {
+      TrainingPartition.Start = classificationProblemData.TrainingPartition.Start;
+      TrainingPartition.End = classificationProblemData.TrainingPartition.End;
+      TestPartition.Start = classificationProblemData.TestPartition.Start;
+      TestPartition.End = classificationProblemData.TestPartition.End;
+
+      for (int i = 0; i < classificationProblemData.ClassNames.Count(); i++)
+        ClassNamesParameter.Value[i, 0] = classificationProblemData.ClassNames.ElementAt(i);
+
+      for (int i = 0; i < Classes; i++) {
+        for (int j = 0; j < Classes; j++) {
+          ClassificationPenaltiesParameter.Value[i, j] = classificationProblemData.GetClassificationPenalty(ClassValuesCache[i], ClassValuesCache[j]);
+        }
+      }
+    }
+
     public ClassificationProblemData(Dataset dataset, IEnumerable<string> allowedInputVariables, string targetVariable)
       : base(dataset, allowedInputVariables) {
@@ -286 +300 @@
       Parameters.Add(new FixedValueParameter<DoubleMatrix>(ClassificationPenaltiesParameterName, ""));
 
+      RegisterParameterEvents();
       ResetTargetVariableDependentMembers();
-      RegisterParameterEvents();
-    }
-
-    private static IEnumerable<string> CheckVariablesForPossibleTargetVariables(Dataset dataset) {
+    }
+
+    public static IEnumerable<string> CheckVariablesForPossibleTargetVariables(Dataset dataset) {
       int maxSamples = Math.Min(InspectedRowsToDetermineTargets, dataset.Rows);
       var validTargetVariables = (from v in dataset.DoubleVariables
@@ -310 +324 @@
       DeregisterParameterEvents();
 
-      classNames = null;
       ((IStringConvertibleMatrix)ClassNamesParameter.Value).Columns = 1;
-      ((IStringConvertibleMatrix)ClassNamesParameter.Value).Rows = ClassValues.Count;
+      ((IStringConvertibleMatrix)ClassNamesParameter.Value).Rows = ClassValuesCache.Count;
       for (int i = 0; i < Classes; i++)
-        ClassNamesParameter.Value[i, 0] = "Class " + ClassValues[i];
+        ClassNamesParameter.Value[i, 0] = "Class " + ClassValuesCache[i];
       ClassNamesParameter.Value.ColumnNames = new List<string>() { "ClassNames" };
       ClassNamesParameter.Value.RowNames = ClassValues.Select(s => "ClassValue: " + s);
 
-      classificationPenaltiesCache.Clear();
-      ((ValueParameter<DoubleMatrix>)ClassificationPenaltiesParameter).ReactOnValueToStringChangedAndValueItemImageChanged = false;
       ((IStringConvertibleMatrix)ClassificationPenaltiesParameter.Value).Rows = Classes;
       ((IStringConvertibleMatrix)ClassificationPenaltiesParameter.Value).Columns = Classes;
@@ -330 +341 @@
         }
       }
-      ((ValueParameter<DoubleMatrix>)ClassificationPenaltiesParameter).ReactOnValueToStringChangedAndValueItemImageChanged = true;
       RegisterParameterEvents();
     }
 
     public string GetClassName(double classValue) {
-      if (!ClassValues.Contains(classValue)) throw new ArgumentException();
-      int index = ClassValues.IndexOf(classValue);
-      return ClassNames[index];
+      if (!ClassValuesCache.Contains(classValue)) throw new ArgumentException();
+      int index = ClassValuesCache.IndexOf(classValue);
+      return ClassNamesCache[index];
     }
     public double GetClassValue(string className) {
-      if (!ClassNames.Contains(className)) throw new ArgumentException();
-      int index = ClassNames.IndexOf(className);
-      return ClassValues[index];
+      if (!ClassNamesCache.Contains(className)) throw new ArgumentException();
+      int index = ClassNamesCache.IndexOf(className);
+      return ClassValuesCache[index];
     }
     public void SetClassName(double classValue, string className) {
-      if (!classValues.Contains(classValue)) throw new ArgumentException();
-      int index = ClassValues.IndexOf(classValue);
-      ClassNames[index] = className;
+      if (!ClassValuesCache.Contains(classValue)) throw new ArgumentException();
+      int index = ClassValuesCache.IndexOf(classValue);
       ClassNamesParameter.Value[index, 0] = className;
+      // updating of class names cache is not necessary here as the parameter value fires a changed event which updates the cache
     }
 
@@ -355 +365 @@
     }
     public double GetClassificationPenalty(double correctClassValue, double estimatedClassValue) {
-      var key = Tuple.Create(correctClassValue, estimatedClassValue);
-      if (!classificationPenaltiesCache.ContainsKey(key)) {
-        int correctClassIndex = ClassValues.IndexOf(correctClassValue);
-        int estimatedClassIndex = ClassValues.IndexOf(estimatedClassValue);
-        classificationPenaltiesCache[key] = ClassificationPenaltiesParameter.Value[correctClassIndex, estimatedClassIndex];
-      }
-      return classificationPenaltiesCache[key];
+      int correctClassIndex = ClassValuesCache.IndexOf(correctClassValue);
+      int estimatedClassIndex = ClassValuesCache.IndexOf(estimatedClassValue);
+      return ClassificationPenaltiesParameter.Value[correctClassIndex, estimatedClassIndex];
     }
     public void SetClassificationPenalty(string correctClassName, string estimatedClassName, double penalty) {
@@ -367 +373 @@
     }
     public void SetClassificationPenalty(double correctClassValue, double estimatedClassValue, double penalty) {
-      var key = Tuple.Create(correctClassValue, estimatedClassValue);
-      int correctClassIndex = ClassValues.IndexOf(correctClassValue);
-      int estimatedClassIndex = ClassValues.IndexOf(estimatedClassValue);
+      int correctClassIndex = ClassValuesCache.IndexOf(correctClassValue);
+      int estimatedClassIndex = ClassValuesCache.IndexOf(estimatedClassValue);
 
       ClassificationPenaltiesParameter.Value[correctClassIndex, estimatedClassIndex] = penalty;
@@ -378 +383 @@
       TargetVariableParameter.ValueChanged += new EventHandler(TargetVariableParameter_ValueChanged);
       ClassNamesParameter.Value.Reset += new EventHandler(Parameter_ValueChanged);
-      ClassNamesParameter.Value.ItemChanged += new EventHandler<EventArgs<int, int>>(MatrixParameter_ItemChanged);
+      ClassNamesParameter.Value.ItemChanged += new EventHandler<EventArgs<int, int>>(Parameter_ValueChanged);
+      ClassificationPenaltiesParameter.Value.ItemChanged += new EventHandler<EventArgs<int, int>>(Parameter_ValueChanged);
       ClassificationPenaltiesParameter.Value.Reset += new EventHandler(Parameter_ValueChanged);
-      ClassificationPenaltiesParameter.Value.ItemChanged += new EventHandler<EventArgs<int, int>>(MatrixParameter_ItemChanged);
     }
     private void DeregisterParameterEvents() {
       TargetVariableParameter.ValueChanged -= new EventHandler(TargetVariableParameter_ValueChanged);
       ClassNamesParameter.Value.Reset -= new EventHandler(Parameter_ValueChanged);
-      ClassNamesParameter.Value.ItemChanged -= new EventHandler<EventArgs<int, int>>(MatrixParameter_ItemChanged);
+      ClassNamesParameter.Value.ItemChanged -= new EventHandler<EventArgs<int, int>>(Parameter_ValueChanged);
+      ClassificationPenaltiesParameter.Value.ItemChanged -= new EventHandler<EventArgs<int, int>>(Parameter_ValueChanged);
       ClassificationPenaltiesParameter.Value.Reset -= new EventHandler(Parameter_ValueChanged);
-      ClassificationPenaltiesParameter.Value.ItemChanged -= new EventHandler<EventArgs<int, int>>(MatrixParameter_ItemChanged);
     }
 
     private void TargetVariableParameter_ValueChanged(object sender, EventArgs e) {
-      classValues = null;
+      classValuesCache = null;
+      classNamesCache = null;
       ResetTargetVariableDependentMembers();
       OnChanged();
     }
     private void Parameter_ValueChanged(object sender, EventArgs e) {
-      OnChanged();
-    }
-    private void MatrixParameter_ItemChanged(object sender, EventArgs<int, int> e) {
+      classNamesCache = null;
+      ClassificationPenaltiesParameter.Value.RowNames = ClassNames.Select(name => "Actual " + name);
+      ClassificationPenaltiesParameter.Value.ColumnNames = ClassNames.Select(name => "Estimated " + name);
       OnChanged();
     }

branches/OaaS/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationSolution.cs

@@ -45 +45 @@
       : base(model, problemData) {
       evaluationCache = new Dictionary<int, double>(problemData.Dataset.Rows);
+      CalculateClassificationResults();
     }
 

branches/OaaS/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationSolutionBase.cs

@@ -85 +85 @@
     }
 
-    protected void CalculateResults() {
+    protected void CalculateClassificationResults() {
       double[] estimatedTrainingClassValues = EstimatedTrainingClassValues.ToArray(); // cache values
       double[] originalTrainingClassValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndices).ToArray();
@@ -114 +114 @@
 
     public abstract IEnumerable<double> GetEstimatedClassValues(IEnumerable<int> rows);
+
+    protected override void RecalculateResults() {
+      CalculateClassificationResults();
+    }
   }
 }

branches/OaaS/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/DiscriminantFunctionClassificationModel.cs

@@ -33 +33 @@
   [StorableClass]
   [Item("DiscriminantFunctionClassificationModel", "Represents a classification model that uses a discriminant function and classification thresholds.")]
-  public abstract class DiscriminantFunctionClassificationModel : NamedItem, IDiscriminantFunctionClassificationModel {
+  public class DiscriminantFunctionClassificationModel : NamedItem, IDiscriminantFunctionClassificationModel {
     [Storable]
     private IRegressionModel model;
+    public IRegressionModel Model {
+      get { return model; }
+      private set { model = value; }
+    }
 
     [Storable]
@@ -51 +55 @@
     }
 
+    private IDiscriminantFunctionThresholdCalculator thresholdCalculator;
+    [Storable]
+    public IDiscriminantFunctionThresholdCalculator ThresholdCalculator {
+      get { return thresholdCalculator; }
+      private set { thresholdCalculator = value; }
+    }
+
 
     [StorableConstructor]
@@ -61 +72 @@
     }
 
-    public DiscriminantFunctionClassificationModel(IRegressionModel model)
+    public DiscriminantFunctionClassificationModel(IRegressionModel model, IDiscriminantFunctionThresholdCalculator thresholdCalculator)
       : base() {
       this.name = ItemName;
       this.description = ItemDescription;
       this.model = model;
-      this.classValues = new double[] { 0.0 };
-      this.thresholds = new double[] { double.NegativeInfinity };
+      this.classValues = new double[0];
+      this.thresholds = new double[0];
+      this.thresholdCalculator = thresholdCalculator;
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      if (ThresholdCalculator == null) ThresholdCalculator = new AccuracyMaximizationThresholdCalculator();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new DiscriminantFunctionClassificationModel(this, cloner);
     }
 
@@ -80 +101 @@
     }
 
+    public virtual void RecalculateModelParameters(IClassificationProblemData problemData, IEnumerable<int> rows) {
+      double[] classValues;
+      double[] thresholds;
+      var targetClassValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows);
+      var estimatedTrainingValues = GetEstimatedValues(problemData.Dataset, rows);
+      thresholdCalculator.Calculate(problemData, estimatedTrainingValues, targetClassValues, out classValues, out thresholds);
+      SetThresholdsAndClassValues(thresholds, classValues);
+    }
+
+
     public IEnumerable<double> GetEstimatedValues(Dataset dataset, IEnumerable<int> rows) {
       return model.GetEstimatedValues(dataset, rows);
@@ -85 +116 @@
 
     public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
+      if (!Thresholds.Any() && !ClassValues.Any()) throw new ArgumentException("No thresholds and class values were set for the current classification model.");
       foreach (var x in GetEstimatedValues(dataset, rows)) {
         int classIndex = 0;
@@ -103 +135 @@
     #endregion
 
-    public abstract IDiscriminantFunctionClassificationSolution CreateDiscriminantFunctionClassificationSolution(IClassificationProblemData problemData);
-    public abstract IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData);
+    public virtual IDiscriminantFunctionClassificationSolution CreateDiscriminantFunctionClassificationSolution(IClassificationProblemData problemData) {
+      return new DiscriminantFunctionClassificationSolution(this, new ClassificationProblemData(problemData));
+    }
+
+    public virtual IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return CreateDiscriminantFunctionClassificationSolution(problemData);
+    }
   }
 }

branches/OaaS/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/DiscriminantFunctionClassificationSolution.cs

@@ -32 +32 @@
   [StorableClass]
   [Item("DiscriminantFunctionClassificationSolution", "Represents a classification solution that uses a discriminant function and classification thresholds.")]
-  public abstract class DiscriminantFunctionClassificationSolution : DiscriminantFunctionClassificationSolutionBase {
+  public class DiscriminantFunctionClassificationSolution : DiscriminantFunctionClassificationSolutionBase {
     protected readonly Dictionary<int, double> valueEvaluationCache;
     protected readonly Dictionary<int, double> classValueEvaluationCache;
@@ -47 +47 @@
       classValueEvaluationCache = new Dictionary<int, double>(original.classValueEvaluationCache);
     }
-    protected DiscriminantFunctionClassificationSolution(IDiscriminantFunctionClassificationModel model, IClassificationProblemData problemData)
+    public DiscriminantFunctionClassificationSolution(IDiscriminantFunctionClassificationModel model, IClassificationProblemData problemData)
       : base(model, problemData) {
       valueEvaluationCache = new Dictionary<int, double>();
       classValueEvaluationCache = new Dictionary<int, double>();
+      CalculateRegressionResults();
+      CalculateClassificationResults();
+    }
 
-      SetAccuracyMaximizingThresholds();
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new DiscriminantFunctionClassificationSolution(this, cloner);
     }
 

branches/OaaS/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/DiscriminantFunctionClassificationSolutionBase.cs

@@ -85 +85 @@
       Add(new Result(TrainingRSquaredResultName, "Squared Pearson's correlation coefficient of the model output and the actual values on the training partition", new DoubleValue()));
       Add(new Result(TestRSquaredResultName, "Squared Pearson's correlation coefficient of the model output and the actual values on the test partition", new DoubleValue()));
-
       RegisterEventHandler();
     }
@@ -92 +91 @@
     private void AfterDeserialization() {
       RegisterEventHandler();
-    }
-
-    protected override void OnModelChanged() {
-      DeregisterEventHandler();
-      SetAccuracyMaximizingThresholds();
-      RegisterEventHandler();
-      base.OnModelChanged();
     }
 
@@ -137 +129 @@
     }
 
-    public void SetAccuracyMaximizingThresholds() {
-      double[] classValues;
-      double[] thresholds;
-      var targetClassValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndices);
-      AccuracyMaximizationThresholdCalculator.CalculateThresholds(ProblemData, EstimatedTrainingValues, targetClassValues, out classValues, out thresholds);
-
-      Model.SetThresholdsAndClassValues(thresholds, classValues);
-    }
-
-    public void SetClassDistibutionCutPointThresholds() {
-      double[] classValues;
-      double[] thresholds;
-      var targetClassValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndices);
-      NormalDistributionCutPointsThresholdCalculator.CalculateThresholds(ProblemData, EstimatedTrainingValues, targetClassValues, out classValues, out thresholds);
-
-      Model.SetThresholdsAndClassValues(thresholds, classValues);
-    }
-
     protected virtual void OnModelThresholdsChanged(EventArgs e) {
-      CalculateResults();
-      CalculateRegressionResults();
+      OnModelChanged();
     }
 
@@ -165 +138 @@
 
     public abstract IEnumerable<double> GetEstimatedValues(IEnumerable<int> rows);
+
+    protected override void RecalculateResults() {
+      base.RecalculateResults();
+      CalculateRegressionResults();
+    }
   }
 }

branches/OaaS/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ThresholdCalculators/AccuracyMaximizationThresholdCalculator.cs

@@ -53 +53 @@
 
     public static void CalculateThresholds(IClassificationProblemData problemData, IEnumerable<double> estimatedValues, IEnumerable<double> targetClassValues, out double[] classValues, out double[] thresholds) {
-      int slices = 100;
-      double minThresholdInc = 10e-5; // necessary to prevent infinite loop when maxEstimated - minEstimated is effectively zero (constant model)
+      const int slices = 100;
+      const double minThresholdInc = 10e-5; // necessary to prevent infinite loop when maxEstimated - minEstimated is effectively zero (constant model)
       List<double> estimatedValuesList = estimatedValues.ToList();
       double maxEstimatedValue = estimatedValuesList.Max();
@@ -61 +61 @@
       var estimatedAndTargetValuePairs =
         estimatedValuesList.Zip(targetClassValues, (x, y) => new { EstimatedValue = x, TargetClassValue = y })
-        .OrderBy(x => x.EstimatedValue)
-        .ToList();
+        .OrderBy(x => x.EstimatedValue).ToList();
 
-      classValues = problemData.ClassValues.OrderBy(x => x).ToArray();
+      classValues = estimatedAndTargetValuePairs.GroupBy(x => x.TargetClassValue)
+        .Select(x => new { Median = x.Select(y => y.EstimatedValue).Median(), Class = x.Key })
+        .OrderBy(x => x.Median).Select(x => x.Class).ToArray();
+
       int nClasses = classValues.Length;
       thresholds = new double[nClasses];
       thresholds[0] = double.NegativeInfinity;
-      // thresholds[thresholds.Length - 1] = double.PositiveInfinity;
 
       // incrementally calculate accuracy of all possible thresholds
@@ -85 +86 @@
             //all positives
             if (pair.TargetClassValue.IsAlmost(classValues[i - 1])) {
-              if (pair.EstimatedValue > lowerThreshold && pair.EstimatedValue < actualThreshold)
+              if (pair.EstimatedValue > lowerThreshold && pair.EstimatedValue <= actualThreshold)
                 //true positive
-                classificationScore += problemData.GetClassificationPenalty(classValues[i - 1], classValues[i - 1]);
+                classificationScore += problemData.GetClassificationPenalty(pair.TargetClassValue, pair.TargetClassValue);
               else
                 //false negative
-                classificationScore += problemData.GetClassificationPenalty(classValues[i], classValues[i - 1]);
+                classificationScore += problemData.GetClassificationPenalty(pair.TargetClassValue, classValues[i]);
             }
               //all negatives
             else {
-              if (pair.EstimatedValue > lowerThreshold && pair.EstimatedValue < actualThreshold)
-                //false positive
-                classificationScore += problemData.GetClassificationPenalty(classValues[i - 1], classValues[i]);
-              else
-                //true negative, consider only upper class
-                classificationScore += problemData.GetClassificationPenalty(classValues[i], classValues[i]);
+              //false positive
+              if (pair.EstimatedValue > lowerThreshold && pair.EstimatedValue <= actualThreshold)
+                classificationScore += problemData.GetClassificationPenalty(pair.TargetClassValue, classValues[i - 1]);
+              else if (pair.EstimatedValue <= lowerThreshold)
+                classificationScore += problemData.GetClassificationPenalty(pair.TargetClassValue, classValues[i - 2]);
+              else if (pair.EstimatedValue > actualThreshold) {
+                if (pair.TargetClassValue < classValues[i - 1]) //negative in wrong class, consider upper class
+                  classificationScore += problemData.GetClassificationPenalty(pair.TargetClassValue, classValues[i]);
+                else //true negative, must be optimized by the other thresholds
+                  classificationScore += problemData.GetClassificationPenalty(pair.TargetClassValue, pair.TargetClassValue);
+              }
             }
           }

branches/OaaS/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ThresholdCalculators/NormalDistributionCutPointsThresholdCalculator.cs

@@ -53 +53 @@
 
     public static void CalculateThresholds(IClassificationProblemData problemData, IEnumerable<double> estimatedValues, IEnumerable<double> targetClassValues, out double[] classValues, out double[] thresholds) {
-      double maxEstimatedValue = estimatedValues.Max();
-      double minEstimatedValue = estimatedValues.Min();
       var estimatedTargetValues = Enumerable.Zip(estimatedValues, targetClassValues, (e, t) => new { EstimatedValue = e, TargetValue = t }).ToList();
+      double estimatedValuesRange = estimatedValues.Range();
 
       Dictionary<double, double> classMean = new Dictionary<double, double>();
@@ -72 +71 @@
         }
       }
+
       double[] originalClasses = classMean.Keys.OrderBy(x => x).ToArray();
       int nClasses = originalClasses.Length;
@@ -82 +82 @@
           // calculate all thresholds
           CalculateCutPoints(classMean[class0], classStdDev[class0], classMean[class1], classStdDev[class1], out x1, out x2);
-          if (!thresholdList.Any(x => x.IsAlmost(x1))) thresholdList.Add(x1);
-          if (!thresholdList.Any(x => x.IsAlmost(x2))) thresholdList.Add(x2);
+
+          // if the two cut points are too close (for instance because the stdDev=0)
+          // then move them by 0.1% of the range of estimated values
+          if (x1.IsAlmost(x2)) {
+            x1 -= 0.001 * estimatedValuesRange;
+            x2 += 0.001 * estimatedValuesRange;
+          }
+          if (!double.IsInfinity(x1) && !thresholdList.Any(x => x.IsAlmost(x1))) thresholdList.Add(x1);
+          if (!double.IsInfinity(x2) && !thresholdList.Any(x => x.IsAlmost(x2))) thresholdList.Add(x2);
         }
       }
       thresholdList.Sort();
+
+      // add small value and large value for the calculation of most influential class in each thresholded section
       thresholdList.Insert(0, double.NegativeInfinity);
-
-      // determine class values for each partition separated by a threshold by calculating the density of all class distributions
-      // all points in the partition are classified as the class with the maximal density in the parition
-      List<double> classValuesList = new List<double>();
-      for (int i = 0; i < thresholdList.Count; i++) {
-        double m;
-        if (double.IsNegativeInfinity(thresholdList[i])) {
-          m = thresholdList[i + 1] - 1.0; // smaller than the smalles non-infinity threshold
-        } else if (i == thresholdList.Count - 1) {
-          // last threshold
-          m = thresholdList[i] + 1.0; // larger than the last threshold
-        } else {
-          m = thresholdList[i] + (thresholdList[i + 1] - thresholdList[i]) / 2.0; // middle of partition
-        }
-
-        // determine class with maximal probability density in m
-        double maxDensity = double.MinValue;
-        double maxDensityClassValue = -1;
-        foreach (var classValue in originalClasses) {
-          double density = NormalDensity(m, classMean[classValue], classStdDev[classValue]);
+      thresholdList.Add(double.PositiveInfinity);
+
+
+      // find the most likely class for the points between thresholds m
+      List<double> filteredThresholds = new List<double>();
+      List<double> filteredClassValues = new List<double>();
+      for (int i = 0; i < thresholdList.Count - 1; i++) {
+        // determine class with maximal density mass between the thresholds
+        double maxDensity = DensityMass(thresholdList[i], thresholdList[i + 1], classMean[originalClasses[0]], classStdDev[originalClasses[0]]);
+        double maxDensityClassValue = originalClasses[0];
+        foreach (var classValue in originalClasses.Skip(1)) {
+          double density = DensityMass(thresholdList[i], thresholdList[i + 1], classMean[classValue], classStdDev[classValue]);
           if (density > maxDensity) {
             maxDensity = density;
@@ -113 +114 @@
           }
         }
-        classValuesList.Add(maxDensityClassValue);
-      }
-
-      // only keep thresholds at which the class changes 
-      // class B overrides threshold s. So only thresholds r and t are relevant and have to be kept
-      // 
-      //      A    B  C
-      //       /\  /\/\        
-      //      / r\/ /\t\       
-      //     /   /\/  \ \      
-      //    /   / /\s  \ \     
-      //  -/---/-/ -\---\-\----
-      List<double> filteredThresholds = new List<double>();
-      List<double> filteredClassValues = new List<double>();
-      filteredThresholds.Add(thresholdList[0]);
-      filteredClassValues.Add(classValuesList[0]);
-      for (int i = 0; i < classValuesList.Count - 1; i++) {
-        if (classValuesList[i] != classValuesList[i + 1]) {
-          filteredThresholds.Add(thresholdList[i + 1]);
-          filteredClassValues.Add(classValuesList[i + 1]);
-        }
-      }
+        if (maxDensity > double.NegativeInfinity &&
+          (filteredClassValues.Count == 0 || !maxDensityClassValue.IsAlmost(filteredClassValues.Last()))) {
+          filteredThresholds.Add(thresholdList[i]);
+          filteredClassValues.Add(maxDensityClassValue);
+        }
+      }
+
+      if (filteredThresholds.Count == 0 || !double.IsNegativeInfinity(filteredThresholds.First())) {
+        // this happens if there are no thresholds (distributions for all classes are exactly the same)
+        // or when the CDF up to the first threshold is zero
+        // -> all samples should be classified as the class with the most observations
+        // group observations by target class and select the class with largest count 
+        double mostFrequentClass = targetClassValues.GroupBy(c => c)
+                              .OrderBy(g => g.Count())
+                              .Last().Key;
+        filteredThresholds.Insert(0, double.NegativeInfinity);
+        filteredClassValues.Insert(0, mostFrequentClass);
+      }
+
       thresholds = filteredThresholds.ToArray();
       classValues = filteredClassValues.ToArray();
     }
 
-    private static double NormalDensity(double x, double mu, double sigma) {
+    private static double sqr2 = Math.Sqrt(2.0);
+    // returns the density function of the standard normal distribution at x
+    private static double NormalCDF(double x) {
+      return 0.5 * (1 + alglib.errorfunction(x / sqr2));
+    }
+
+    // approximation of the log of the normal cummulative distribution from the lightspeed toolbox by Tom Minka
+    // http://research.microsoft.com/en-us/um/people/minka/software/lightspeed/
+    private static double[] c = new double[] { -1, 5 / 2.0, -37 / 3.0, 353 / 4.0, -4081 / 5.0, 55205 / 6.0, -854197 / 7.0 };
+    private static double LogNormalCDF(double x) {
+      if (x >= -6.5)
+        // calculate the log directly if x is large enough
+        return Math.Log(NormalCDF(x));
+      else {
+        double z = Math.Pow(x, -2);
+        // asymptotic series for logcdf
+        double y = z * (c[0] + z * (c[1] + z * (c[2] + z * (c[3] + z * (c[4] + z * (c[5] + z * c[6]))))));
+        return y - 0.5 * Math.Log(2 * Math.PI) - 0.5 * x * x - Math.Log(-x);
+      }
+    }
+
+    // determines the value NormalCDF(mu,sigma, upper)  - NormalCDF(mu, sigma, lower)
+    // = the integral of the PDF of N(mu, sigma) in the range [lower, upper]
+    private static double DensityMass(double lower, double upper, double mu, double sigma) {
       if (sigma.IsAlmost(0.0)) {
-        if (x.IsAlmost(mu)) return 1.0; else return 0.0;
+        if (lower < mu && mu < upper) return 0.0; // all mass is between lower and upper
+        else return double.NegativeInfinity; // no mass is between lower and upper
+      }
+
+      if (lower > mu) {
+        return DensityMass(-upper, -lower, -mu, sigma);
+      }
+
+      upper = (upper - mu) / sigma;
+      lower = (lower - mu) / sigma;
+      if (double.IsNegativeInfinity(lower)) return LogNormalCDF(upper);
+
+      return LogNormalCDF(upper) + Math.Log(1 - Math.Exp(LogNormalCDF(lower) - LogNormalCDF(upper)));
+    }
+
+    // Calculates the points x1 and x2 where the distributions N(m1, s1) == N(m2,s2). 
+    // In the general case there should be two cut points. If either s1 or s2 is 0 then x1==x2. 
+    // If both s1 and s2 are zero than there are no cut points but we should return something reasonable (e.g. (m1 + m2) / 2) then.
+    private static void CalculateCutPoints(double m1, double s1, double m2, double s2, out double x1, out double x2) {
+      if (s1.IsAlmost(s2)) {
+        if (m1.IsAlmost(m2)) {
+          x1 = double.NegativeInfinity;
+          x2 = double.NegativeInfinity;
+        } else {
+          // s1==s2 and m1 != m2
+          // return something reasonable. cut point should be half way between m1 and m2
+          x1 = (m1 + m2) / 2;
+          x2 = double.NegativeInfinity;
+        }
+      } else if (s1.IsAlmost(0.0)) {
+        // when s1 is 0.0 the cut points are exactly at m1 ...
+        x1 = m1;
+        x2 = m1;
+      } else if (s2.IsAlmost(0.0)) {
+        // ... same for s2
+        x1 = m2;
+        x2 = m2;
       } else {
-        return (1.0 / Math.Sqrt(2.0 * Math.PI * sigma * sigma)) * Math.Exp(-((x - mu) * (x - mu)) / (2.0 * sigma * sigma));
-      }
-    }
-
-    private static void CalculateCutPoints(double m1, double s1, double m2, double s2, out double x1, out double x2) {
-      double a = (s1 * s1 - s2 * s2);
-      x1 = -(-m2 * s1 * s1 + m1 * s2 * s2 + Math.Sqrt(s1 * s1 * s2 * s2 * ((m1 - m2) * (m1 - m2) + 2.0 * (-s1 * s1 + s2 * s2) * Math.Log(s2 / s1)))) / a;
-      x2 = (m2 * s1 * s1 - m1 * s2 * s2 + Math.Sqrt(s1 * s1 * s2 * s2 * ((m1 - m2) * (m1 - m2) + 2.0 * (-s1 * s1 + s2 * s2) * Math.Log(s2 / s1)))) / a;
+        if (s2 < s1) {
+          // make sure s2 is the larger std.dev.
+          CalculateCutPoints(m2, s2, m1, s1, out x1, out x2);
+        } else {
+          // general case
+          // calculate the solutions x1, x2 where N(m1,s1) == N(m2,s2)
+          double g = Math.Sqrt(2 * s2 * s2 * Math.Log(s2 / s1) - 2 * s1 * s1 * Math.Log(s2 / s1) - 2 * m1 * m2 + m1 * m1 + m2 * m2);
+          double s = (s1 * s1 - s2 * s2);
+          x1 = (m2 * s1 * s1 - m1 * s2 * s2 + s1 * s2 * g) / s;
+          x2 = -(m1 * s2 * s2 - m2 * s1 * s1 + s1 * s2 * g) / s;
+        }
+      }
     }
   }

@@ -21 +21 @@
 protoc.exe
 _ReSharper.HeuristicLab 3.3 Tests
+Google.ProtocolBuffers-2.4.1.473.dll
 packages