Changeset 4309 for branches/DataAnalysis/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/OverfittingAnalyzer.cs

Timestamp:

08/24/10 19:25:11 (14 years ago)

Author:

gkronber

Message:

Exploring overfitting countermeasures. #1142

File:

• branches/DataAnalysis/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/OverfittingAnalyzer.cs (modified) (3 diffs)

branches/DataAnalysis/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/OverfittingAnalyzer.cs

@@ -218 +218 @@
       ItemArray<DoubleValue> validationQualities = ValidationQualityParameter.ActualValue;
 
-      string targetVariable = ProblemData.TargetVariable.Value;
-
-      // select a random subset of rows in the validation set
-      int validationStart = ValidiationSamplesStart.Value;
-      int validationEnd = ValidationSamplesEnd.Value;
-      int seed = Random.Next();
-      int count = (int)((validationEnd - validationStart) * RelativeNumberOfEvaluatedSamples.Value);
-      if (count == 0) count = 1;
-      IEnumerable<int> rows = RandomEnumerable.SampleRandomNumbers(seed, validationStart, validationEnd, count);
-
-      double upperEstimationLimit = UpperEstimationLimit != null ? UpperEstimationLimit.Value : double.PositiveInfinity;
-      double lowerEstimationLimit = LowerEstimationLimit != null ? LowerEstimationLimit.Value : double.NegativeInfinity;
+      //string targetVariable = ProblemData.TargetVariable.Value;
+
+      //// select a random subset of rows in the validation set
+      //int validationStart = ValidiationSamplesStart.Value;
+      //int validationEnd = ValidationSamplesEnd.Value;
+      //int seed = Random.Next();
+      //int count = (int)((validationEnd - validationStart) * RelativeNumberOfEvaluatedSamples.Value);
+      //if (count == 0) count = 1;
+      //IEnumerable<int> rows = RandomEnumerable.SampleRandomNumbers(seed, validationStart, validationEnd, count);
+
+      //double upperEstimationLimit = UpperEstimationLimit != null ? UpperEstimationLimit.Value : double.PositiveInfinity;
+      //double lowerEstimationLimit = LowerEstimationLimit != null ? LowerEstimationLimit.Value : double.NegativeInfinity;
 
       //double bestQuality = Maximization.Value ? double.NegativeInfinity : double.PositiveInfinity;
@@ -250 +250 @@
       //if (RelativeValidationQualityParameter.ActualValue == null) {
       // first call initialize the relative quality using the difference between average training and validation quality
-      double avgTrainingQuality = qualities.Select(x => x.Value).Median();
-      double avgValidationQuality = validationQualities.Select(x => x.Value).Median();
+      double avgTrainingQuality = qualities.Select(x => x.Value).Average();
+      double avgValidationQuality = validationQualities.Select(x => x.Value).Average();
 
       if (Maximization.Value)
@@ -284 +284 @@
       bool overfitting =
         avgTrainingQuality > InitialTrainingQualityParameter.ActualValue.Value &&  // better on training than in initial generation
+        // RelativeValidationQualityParameter.ActualValue.Value < 0.0 && // validation quality is worse than training quality
         r < CorrelationLimitParameter.ActualValue.Value;  // low correlation between training and validation quality
 
-      //// if validation quality is within a certain margin of percentage deviation (default -5% .. 5%) then there is no overfitting
-      //// correlation is also bad when underfitting but validation quality cannot be a lot larger than training quality if overfitting
-      //(RelativeValidationQualityParameter.ActualValue.Value > RelativeValidationQualityUpperLimitParameter.ActualValue.Value || // better on training than on validation
-      // RelativeValidationQualityParameter.ActualValue.Value < RelativeValidationQualityLowerLimitParameter.ActualValue.Value); // better on training than on validation
 
       OverfittingParameter.ActualValue = new BoolValue(overfitting);