Changeset 4275 for branches/DataAnalysis/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/OverfittingAnalyzer.cs

Timestamp:

08/22/10 19:06:32 (14 years ago)

Author:

gkronber

Message:

Improved overfitting analyzer #1142

File:

• branches/DataAnalysis/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/OverfittingAnalyzer.cs (modified) (5 diffs)

branches/DataAnalysis/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/OverfittingAnalyzer.cs

@@ -112 +112 @@
     public ILookupParameter<DoubleValue> InitialTrainingQualityParameter {
       get { return (ILookupParameter<DoubleValue>)Parameters["InitialTrainingQuality"]; }
+    }
+    public ILookupParameter<DoubleMatrix> TrainingAndValidationQualitiesParameter {
+      get { return (ILookupParameter<DoubleMatrix>)Parameters["TrainingAndValidationQualities"]; }
+    }
+    public IValueLookupParameter<DoubleValue> PercentileParameter {
+      get { return (IValueLookupParameter<DoubleValue>)Parameters["Percentile"]; }
     }
     #endregion
@@ -173 +179 @@
       Parameters.Add(new LookupParameter<ResultCollection>("Results"));
       Parameters.Add(new LookupParameter<DoubleValue>("InitialTrainingQuality"));
+      Parameters.Add(new LookupParameter<DoubleMatrix>("TrainingAndValidationQualities"));
+      Parameters.Add(new ValueLookupParameter<DoubleValue>("Percentile", new DoubleValue(0.1)));
+
     }
 
@@ -189 +198 @@
       //  Parameters.Add(new ValueLookupParameter<PercentValue>("RelativeValidationQualityLowerLimit", new PercentValue(-0.05)));
       //}
+      if (!Parameters.ContainsKey("TrainingAndValidationQualities")) {
+        Parameters.Add(new LookupParameter<DoubleMatrix>("TrainingAndValidationQualities"));
+      }
+      if (!Parameters.ContainsKey("Percentile")) {
+        Parameters.Add(new ValueLookupParameter<DoubleValue>("Percentile", new DoubleValue(0.1)));
+      }
     }
 
@@ -237 +252 @@
       //}
 
-      // cut away 0.0 values to make the correlation stronger
-      // necessary because R² values of 0.0 are strong outliers
-      //int percentile = (int)Math.Round(0.1 * validationQualities.Count);
-      //double validationCutOffValue = validationQualities.OrderBy(x => x).ElementAt(percentile);
-      //double trainingCutOffValue = qualities.Select(x => x.Value).OrderBy(x => x).ElementAt(percentile);
-      double validationCutOffValue = 0.05;
-      double trainingCutOffValue = validationCutOffValue;
-
-      double[] validationArr = new double[validationQualities.Count];
-      double[] trainingArr = new double[validationQualities.Count];
-      int arrIndex = 0;
-      for (int i = 0; i < validationQualities.Count; i++) {
-        if (validationQualities[i] > validationCutOffValue &&
-            qualities[i].Value > trainingCutOffValue) {
-          validationArr[arrIndex] = validationQualities[i];
-          trainingArr[arrIndex] = qualities[i].Value;
-          arrIndex++;
-        }
-      }
-      double r = alglib.correlation.spearmanrankcorrelation(trainingArr, validationArr, arrIndex);
+      // best first (only for maximization
+      var orderedDistinctPairs = (from index in Enumerable.Range(0, qualities.Length)
+                                  select new { Training = qualities[index].Value, Validation = validationQualities[index] })
+                                 .Distinct()
+                                 .OrderBy(x => -x.Training)
+                                 .ToList();
+
+      int n = (int)Math.Round(PercentileParameter.ActualValue.Value * orderedDistinctPairs.Count);
+
+      double[] validationArr = new double[n];
+      double[] trainingArr = new double[n];
+      //double[,] qualitiesArr = new double[n, 2];
+      for (int i = 0; i < n; i++) {
+        validationArr[i] = orderedDistinctPairs[i].Validation;
+        trainingArr[i] = orderedDistinctPairs[i].Training;
+
+        //qualitiesArr[i, 0] = trainingArr[i];
+        //qualitiesArr[i, 1] = validationArr[i];
+      }
+      double r = alglib.correlation.spearmanrankcorrelation(trainingArr, validationArr, n);
       TrainingValidationQualityCorrelationParameter.ActualValue = new DoubleValue(r);
       if (InitialTrainingQualityParameter.ActualValue == null)
@@ -270 +285 @@
 
       OverfittingParameter.ActualValue = new BoolValue(overfitting);
+      //TrainingAndValidationQualitiesParameter.ActualValue = new DoubleMatrix(qualitiesArr);
       return base.Apply();
     }