Changeset 4271

Timestamp:

08/20/10 12:45:57 (14 years ago)

Author:

gkronber

Message:

Added correlation-based overfitting detection analyzer. #1142

Location:

branches/DataAnalysis

Files:

• HeuristicLab.Problems.DataAnalysis.Regression/3.3/HeuristicLab.Problems.DataAnalysis.Regression-3.3.csproj (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/FixedValidationBestScaledSymbolicRegressionSolutionAnalyzer.cs (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/OverfittingAnalyzer.cs (added)
• HeuristicLab.Problems.DataAnalysis/3.3/Operators/CovariantParsimonyPressure.cs (modified) (7 diffs)

branches/DataAnalysis/HeuristicLab.Problems.DataAnalysis.Regression/3.3/HeuristicLab.Problems.DataAnalysis.Regression-3.3.csproj

@@ -119 +119 @@
     <Compile Include="SupportVectorRegression\SupportVectorRegressionSolution.cs" />
     <Compile Include="Symbolic\Analyzers\BestSymbolicRegressionSolutionAnalyzer.cs" />
+    <Compile Include="Symbolic\Analyzers\OverfittingAnalyzer.cs" />
     <Compile Include="Symbolic\Analyzers\FixedValidationBestScaledSymbolicRegressionSolutionAnalyzer.cs" />
     <Compile Include="Symbolic\Analyzers\ISymbolicRegressionAnalyzer.cs" />

branches/DataAnalysis/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/FixedValidationBestScaledSymbolicRegressionSolutionAnalyzer.cs

@@ -238 +238 @@
 
     [StorableConstructor]
-    private FixedValidationBestScaledSymbolicRegressionSolutionAnalyzer(bool deserializing) : base() { }
+    private FixedValidationBestScaledSymbolicRegressionSolutionAnalyzer(bool deserializing) : base(deserializing) { }
 
     [StorableHook(HookType.AfterDeserialization)]

branches/DataAnalysis/HeuristicLab.Problems.DataAnalysis/3.3/Operators/CovariantParsimonyPressure.cs

@@ -60 +60 @@
       get { return (IValueLookupParameter<IntValue>)Parameters["FirstGenerationParameter"]; }
     }
-    public IValueLookupParameter<BoolValue> AntiOverfitParameter {
-      get { return (IValueLookupParameter<BoolValue>)Parameters["AntiOverfit"]; }
+    public IValueLookupParameter<BoolValue> ApplyParsimonyPressureParameter {
+      get { return (IValueLookupParameter<BoolValue>)Parameters["ApplyParsimonyPressure"]; }
     }
     public ILookupParameter<DataTable> ValidationQualityParameter {
       get { return (ILookupParameter<DataTable>)Parameters["Validation Quality"]; }
     }
-    public ILookupParameter<DoubleValue> CurrentBestValidationQualityParameter {
-      get { return (ILookupParameter<DoubleValue>)Parameters["Current best validation quality"]; }
-    }
-    public ILookupParameter<DoubleValue> BestValidationQualityParameter {
-      get { return (ILookupParameter<DoubleValue>)Parameters["Best solution quality (validation)"]; }
-    }
+    //public ILookupParameter<DoubleValue> CurrentBestValidationQualityParameter {
+    //  get { return (ILookupParameter<DoubleValue>)Parameters["Current best validation quality"]; }
+    //}
+    //public ILookupParameter<DoubleValue> BestValidationQualityParameter {
+    //  get { return (ILookupParameter<DoubleValue>)Parameters["Best solution quality (validation)"]; }
+    //}
     public ILookupParameter<DoubleValue> LengthCorrelationParameter {
       get { return (ILookupParameter<DoubleValue>)Parameters["Correlation(Length, AdjustedFitness)"]; }
@@ -78 +78 @@
       get { return (ILookupParameter<DoubleValue>)Parameters["Correlation(Fitness, AdjustedFitness)"]; }
     }
-    public IValueLookupParameter<IntValue> GenerationSpanParameter {
-      get { return (IValueLookupParameter<IntValue>)Parameters["GenerationSpan"]; }
-    }
-    public IValueLookupParameter<PercentValue> OverfittingLimitParameter {
-      get { return (IValueLookupParameter<PercentValue>)Parameters["OverfittingLimit"]; }
-    }
+    //public IValueLookupParameter<IntValue> GenerationSpanParameter {
+    //  get { return (IValueLookupParameter<IntValue>)Parameters["GenerationSpan"]; }
+    //}
+    //public IValueLookupParameter<PercentValue> OverfittingLimitParameter {
+    //  get { return (IValueLookupParameter<PercentValue>)Parameters["OverfittingLimit"]; }
+    //}
     public IValueLookupParameter<PercentValue> ComplexityAdaptionParameter {
       get { return (IValueLookupParameter<PercentValue>)Parameters["ComplexityAdaption"]; }
@@ -101 +101 @@
       Parameters.Add(new LookupParameter<IntValue>("Generations"));
       Parameters.Add(new ValueLookupParameter<IntValue>("FirstGenerationParameter", new IntValue(5)));
-      Parameters.Add(new ValueLookupParameter<BoolValue>("AntiOverfit", new BoolValue(false)));
+      Parameters.Add(new ValueLookupParameter<BoolValue>("ApplyParsimonyPressure"));
       //Parameters.Add(new LookupParameter<DoubleValue>("Current best validation quality"));
       //Parameters.Add(new LookupParameter<DoubleValue>("Best solution quality (validation)"));
       Parameters.Add(new LookupParameter<DataTable>("Validation Quality"));
       Parameters.Add(new LookupParameter<DataTable>("Qualities"));
-      Parameters.Add(new ValueLookupParameter<IntValue>("GenerationSpan", new IntValue(5)));
-      Parameters.Add(new ValueLookupParameter<PercentValue>("OverfittingLimit", new PercentValue(5)));
+      //Parameters.Add(new ValueLookupParameter<IntValue>("GenerationSpan", new IntValue(5)));
+      //Parameters.Add(new ValueLookupParameter<PercentValue>("OverfittingLimit", new PercentValue(5)));
       Parameters.Add(new ValueLookupParameter<PercentValue>("ComplexityAdaption", new PercentValue(-5)));
       Parameters.Add(new LookupParameter<DoubleValue>("Correlation(Length, AdjustedFitness)"));
@@ -128 +128 @@
         Parameters.Add(new ValueLookupParameter<IntValue>("FirstGenerationParameter", new IntValue(5)));
       }
-      if (!Parameters.ContainsKey("AntiOverfit")) {
-        Parameters.Add(new ValueLookupParameter<BoolValue>("AntiOverfit", new BoolValue(false)));
+      if (!Parameters.ContainsKey("ApplyParsimonyPressure")) {
+        Parameters.Add(new ValueLookupParameter<BoolValue>("ApplyParsimonyPressure"));
       }
       //if (!Parameters.ContainsKey("Current best validation quality")) {
@@ -149 +149 @@
         Parameters.Add(new LookupParameter<DataTable>("Qualities"));
       }
-      if (!Parameters.ContainsKey("GenerationSpan")) {
-        Parameters.Add(new ValueLookupParameter<IntValue>("GenerationSpan", new IntValue(5)));
-      }
-      if (!Parameters.ContainsKey("OverfittingLimit")) {
-        Parameters.Add(new ValueLookupParameter<PercentValue>("OverfittingLimit", new PercentValue(5)));
-      }
       if (!Parameters.ContainsKey("ComplexityAdaption")) {
         Parameters.Add(new ValueLookupParameter<PercentValue>("ComplexityAdaption", new PercentValue(-5)));
@@ -163 +157 @@
       ItemArray<SymbolicExpressionTree> trees = SymbolicExpressionTreeParameter.ActualValue;
       ItemArray<DoubleValue> qualities = QualityParameter.ActualValue;
-      // always apply Parsimony pressure if anti-overfit is false
+      // always apply Parsimony pressure if overfitting has been detected
       // otherwise appliy PP only when we are currently overfitting
       if (GenerationsParameter.ActualValue != null && GenerationsParameter.ActualValue.Value >= FirstGenerationParameter.ActualValue.Value &&
-         (AntiOverfitParameter.ActualValue.Value == false || IsOverfitting())) {
+           ApplyParsimonyPressureParameter.ActualValue.Value == true) {
         var lengths = from tree in trees
                       select tree.Size;
@@ -241 +235 @@
       return base.Apply();
     }
-
-    private bool IsOverfitting() {
-      bool maximization = MaximizationParameter.ActualValue.Value;
-      DataTable trainingQualities = QualitiesParameter.ActualValue;
-      DataTable validationQualities = ValidationQualityParameter.ActualValue;
-      int genSpan = GenerationSpanParameter.ActualValue.Value;
-      if (validationQualities == null || trainingQualities == null) return false;
-      if (validationQualities.Rows["Best solution quality (validation)"].Values.Count < genSpan) return false;
-
-      IEnumerable<double> bestTrainingQualities = trainingQualities.Rows["CurrentBestQuality"].Values;
-      IEnumerable<double> bestValidationQualities = validationQualities.Rows["Current best validation quality"].Values;
-
-      double trainingAvg = bestTrainingQualities.Reverse().Take(genSpan).Average();
-      double validationAvg = bestValidationQualities.Reverse().Take(genSpan).Average();
-
-      double maxPercentDiff = OverfittingLimitParameter.ActualValue.Value;
-
-      double percentDiff = maximization ? trainingAvg / validationAvg - 1 : validationAvg / trainingAvg - 1;
-      return percentDiff > maxPercentDiff;
-    }
   }
 }