Changeset 10570 for branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis

Timestamp:

03/11/14 15:41:18 (10 years ago)

Author:

mkommend

Message:

#1998: Adde missing value handling in new implementation of OneR.

File:

• branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/OneRTest.cs (modified) (6 diffs)

branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/OneRTest.cs

@@ -67 +67 @@
       List<Split> bestSplits = null;
       string bestVariable = string.Empty;
+      double bestMissingValuesClass = double.NaN;
+      var classValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
 
-      var classValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
       foreach (var variable in problemData.AllowedInputVariables) {
         var inputValues = problemData.Dataset.GetDoubleValues(variable, problemData.TrainingIndices);
         var samples = inputValues.Zip(classValues, (i, v) => new Sample(i, v)).OrderBy(s => s.inputValue);
+
+        var missingValuesDistribution = samples.Where(s => double.IsNaN(s.inputValue)).GroupBy(s => s.classValue).ToDictionary(s => s.Key, s => s.Count()).MaxItems(s => s.Value).FirstOrDefault();
 
         //calculate class distributions for all distinct inputValues
@@ -77 +80 @@
         List<double> thresholds = new List<double>();
         double lastValue = double.NaN;
-        foreach (var sample in samples) {
+        foreach (var sample in samples.Where(s => !double.IsNaN(s.inputValue))) {
           if (sample.inputValue > lastValue || double.IsNaN(lastValue)) {
             if (!double.IsNaN(lastValue)) thresholds.Add((lastValue + sample.inputValue) / 2);
@@ -96 +99 @@
         for (int i = 1; i < classDistributions.Count; i++) {
           var samplesInSplit = distribution.Max(d => d.Value);
-          //join splits if too few sample in split or the distributions has the same maximum class value
+          //join splits if there are too few samples in the split or the distributions has the same maximum class value as the current split
           if (samplesInSplit < minBucketSize ||
             classDistributions[i].MaxItems(d => d.Value).Select(d => d.Key).Contains(
@@ -113 +116 @@
         int correctClassified = 0;
         int splitIndex = 0;
-        foreach (var sample in samples) {
+        foreach (var sample in samples.Where(s => !double.IsNaN(s.inputValue))) {
           while (sample.inputValue >= splits[splitIndex].thresholdValue)
             splitIndex++;
           correctClassified += sample.classValue == splits[splitIndex].classValue ? 1 : 0;
         }
+        correctClassified += missingValuesDistribution.Value;
 
         if (correctClassified > bestClassified) {
@@ -123 +127 @@
           bestSplits = splits;
           bestVariable = variable;
+          bestMissingValuesClass = missingValuesDistribution.Value == 0 ? double.NaN : missingValuesDistribution.Key;
         }
       }
@@ -134 +139 @@
       }
 
-      var model = new OneRClassificationModel(bestVariable, bestSplits.Select(s => s.thresholdValue).ToArray(), bestSplits.Select(s => s.classValue).ToArray(), 0);
+      var model = new OneRClassificationModel(bestVariable, bestSplits.Select(s => s.thresholdValue).ToArray(), bestSplits.Select(s => s.classValue).ToArray(), bestMissingValuesClass);
       var solution = new OneRClassificationSolution(model, (IClassificationProblemData)problemData.Clone());