Changeset 4326

Timestamp:

08/26/10 12:43:02 (14 years ago)

Author:

gkronber

Message:

Changed OverfittingAnalyzer to make overfitting boundaries more fuzzy through upper and lower limits for correlations instead of a hard limit. #1142

File:

• branches/DataAnalysis/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/OverfittingAnalyzer.cs (modified) (6 diffs)

branches/DataAnalysis/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/OverfittingAnalyzer.cs

@@ -104 +104 @@
       get { return (ILookupParameter<DoubleValue>)Parameters["TrainingValidationCorrelation"]; }
     }
-    public IValueLookupParameter<DoubleValue> CorrelationLimitParameter {
-      get { return (IValueLookupParameter<DoubleValue>)Parameters["CorrelationLimit"]; }
+    public IValueLookupParameter<DoubleValue> LowerCorrelationLimitParameter {
+      get { return (IValueLookupParameter<DoubleValue>)Parameters["LowerCorrelationLimit"]; }
+    }
+    public IValueLookupParameter<DoubleValue> UpperCorrelationLimitParameter {
+      get { return (IValueLookupParameter<DoubleValue>)Parameters["UpperCorrelationLimit"]; }
     }
     public ILookupParameter<BoolValue> OverfittingParameter {
@@ -179 +182 @@
       //Parameters.Add(new ValueLookupParameter<PercentValue>("RelativeValidationQualityLowerLimit", new PercentValue(-0.05)));
       Parameters.Add(new LookupParameter<DoubleValue>("TrainingValidationCorrelation"));
-      Parameters.Add(new ValueLookupParameter<DoubleValue>("CorrelationLimit", new DoubleValue(0.65)));
+      Parameters.Add(new ValueLookupParameter<DoubleValue>("LowerCorrelationLimit", new DoubleValue(0.65)));
+      Parameters.Add(new ValueLookupParameter<DoubleValue>("UpperCorrelationLimit", new DoubleValue(0.75)));
       Parameters.Add(new LookupParameter<BoolValue>("Overfitting"));
       Parameters.Add(new LookupParameter<ResultCollection>("Results"));
@@ -211 +215 @@
         Parameters.Add(new ScopeTreeLookupParameter<DoubleValue>("ValidationQuality"));
       }
+      if (!Parameters.ContainsKey("LowerCorrelationLimit")) {
+        Parameters.Add(new ValueLookupParameter<DoubleValue>("LowerCorrelationLimit", new DoubleValue(0.65)));
+      }
+      if (!Parameters.ContainsKey("UpperCorrelationLimit")) {
+        Parameters.Add(new ValueLookupParameter<DoubleValue>("UpperCorrelationLimit", new DoubleValue(0.75)));
+      }
+
     }
 
@@ -218 +229 @@
       ItemArray<DoubleValue> validationQualities = ValidationQualityParameter.ActualValue;
 
+      double correlationLimit;
+      if (OverfittingParameter.ActualValue != null && OverfittingParameter.ActualValue.Value) {
+        // if is already overfitting have to reach the upper limit to switch back to non-overfitting state
+        correlationLimit = UpperCorrelationLimitParameter.ActualValue.Value;
+      } else {
+        // if currently in non-overfitting state have to reach to lower limit to switch to overfitting state
+        correlationLimit = LowerCorrelationLimitParameter.ActualValue.Value;
+      }
       //string targetVariable = ProblemData.TargetVariable.Value;
 
@@ -262 +281 @@
       // best first (only for maximization
       var orderedDistinctPairs = (from index in Enumerable.Range(0, qualities.Length)
+                                  where qualities[index].Value > 0.0
                                   select new { Training = qualities[index].Value, Validation = validationQualities[index].Value })
                                  .OrderBy(x => -x.Training)
@@ -285 +305 @@
         avgTrainingQuality > InitialTrainingQualityParameter.ActualValue.Value &&  // better on training than in initial generation
         // RelativeValidationQualityParameter.ActualValue.Value < 0.0 && // validation quality is worse than training quality
-        r < CorrelationLimitParameter.ActualValue.Value;  // low correlation between training and validation quality
+        r < correlationLimit;