Changeset 4272

Timestamp:

08/20/10 17:42:27 (14 years ago)

Author:

gkronber

Message:

Worked on overfitting analyzer and CPP. #1142

Location:

branches/DataAnalysis

Files:

• HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/FixedValidationBestScaledSymbolicRegressionSolutionAnalyzer.cs (modified) (8 diffs)
• HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/OverfittingAnalyzer.cs (modified) (7 diffs)
• HeuristicLab.Problems.DataAnalysis/3.3/Operators/CovariantParsimonyPressure.cs (modified) (4 diffs)

branches/DataAnalysis/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/FixedValidationBestScaledSymbolicRegressionSolutionAnalyzer.cs

@@ -140 +140 @@
       get { return (ILookupParameter<SymbolicRegressionSolution>)Parameters[BestSolutionParameterName]; }
     }
+    public ILookupParameter<SymbolicRegressionSolution> BestTrainingSolutionParameter {
+      get { return (ILookupParameter<SymbolicRegressionSolution>)Parameters["BestTrainingSolution"]; }
+    }
+    public ScopeTreeLookupParameter<DoubleValue> QualityParameter {
+      get { return (ScopeTreeLookupParameter<DoubleValue>)Parameters["Quality"]; }
+    }
+
     public ILookupParameter<IntValue> GenerationsParameter {
       get { return (ILookupParameter<IntValue>)Parameters[GenerationsParameterName]; }
@@ -228 +235 @@
       Parameters.Add(new ValueLookupParameter<DoubleValue>(LowerEstimationLimitParameterName, "The lower estimation limit that was set for the evaluation of the symbolic expression trees."));
       Parameters.Add(new LookupParameter<SymbolicRegressionSolution>(BestSolutionParameterName, "The best symbolic regression solution."));
+      Parameters.Add(new LookupParameter<SymbolicRegressionSolution>("BestTrainingSolution"));
+      Parameters.Add(new ScopeTreeLookupParameter<DoubleValue>("Quality"));
       Parameters.Add(new LookupParameter<IntValue>(GenerationsParameterName, "The number of generations calculated so far."));
       Parameters.Add(new LookupParameter<DoubleValue>(BestSolutionQualityParameterName, "The quality of the best symbolic regression solution."));
@@ -252 +261 @@
         Parameters.Add(new LookupParameter<DataTable>(BestSolutionQualityValuesParameterName));
       }
+      if (!Parameters.ContainsKey("BestTrainingSolution")) {
+        Parameters.Add(new LookupParameter<SymbolicRegressionSolution>("BestTrainingSolution"));
+      }
+      if (!Parameters.ContainsKey("Quality")) {
+        Parameters.Add(new ScopeTreeLookupParameter<DoubleValue>("Quality"));
+      }
       #endregion
     }
 
     public override IOperation Apply() {
-      var trees = SymbolicExpressionTree;
+      ItemArray<SymbolicExpressionTree> trees = SymbolicExpressionTree;
+      ItemArray<DoubleValue> qualities = QualityParameter.ActualValue;
 
       string targetVariable = ProblemData.TargetVariable.Value;
@@ -273 +289 @@
       double bestQuality = Maximization.Value ? double.NegativeInfinity : double.PositiveInfinity;
       SymbolicExpressionTree bestTree = null;
-
-      foreach (var tree in trees) {
+      SymbolicExpressionTree bestTrainingTree = trees[0];
+      double bestTrainingQuality = qualities[0].Value;
+      for (int i = 0; i < trees.Length; i++) {
+        SymbolicExpressionTree tree = trees[i];
         double quality = Evaluator.Evaluate(SymbolicExpressionTreeInterpreter, tree,
           lowerEstimationLimit, upperEstimationLimit,
@@ -285 +303 @@
           bestTree = tree;
         }
-      }
+        if ((Maximization.Value && qualities[i].Value > bestTrainingQuality) ||
+            (!Maximization.Value && qualities[i].Value < bestTrainingQuality)) {
+          bestTrainingQuality = qualities[i].Value;
+          bestTrainingTree = tree;
+        }
+      }
+
+      var scaledBestTrainingTree = GetScaledTree(bestTrainingTree);
+
+      SymbolicRegressionSolution bestTrainingSolution = new SymbolicRegressionSolution(ProblemData,
+        new SymbolicRegressionModel(SymbolicExpressionTreeInterpreter, scaledBestTrainingTree),
+        lowerEstimationLimit, upperEstimationLimit);
+      bestTrainingSolution.Name = "Best solution (training)";
+      bestTrainingSolution.Description = "The solution of the population with the highest fitness";
 
       // if the best validation tree is better than the current best solution => update
@@ -293 +324 @@
         (!Maximization.Value && bestQuality < BestSolutionQuality.Value);
       if (newBest) {
-        // calculate scaling parameters and only for the best tree using the full training set
-        double alpha, beta;
-        int trainingStart = ProblemData.TrainingSamplesStart.Value;
-        int trainingEnd = ProblemData.TrainingSamplesEnd.Value;
-        IEnumerable<int> trainingRows = Enumerable.Range(trainingStart, trainingEnd - trainingStart);
-        IEnumerable<double> originalValues = ProblemData.Dataset.GetEnumeratedVariableValues(targetVariable, trainingRows);
-        IEnumerable<double> estimatedValues = SymbolicExpressionTreeInterpreter.GetSymbolicExpressionTreeValues(bestTree, ProblemData.Dataset, trainingRows);
-
-        SymbolicRegressionScaledMeanSquaredErrorEvaluator.CalculateScalingParameters(originalValues, estimatedValues, out beta, out alpha);
-
-        // scale tree for solution
-        var scaledTree = SymbolicRegressionSolutionLinearScaler.Scale(bestTree, alpha, beta);
+        var scaledTree = GetScaledTree(bestTree);
         var model = new SymbolicRegressionModel((ISymbolicExpressionTreeInterpreter)SymbolicExpressionTreeInterpreter.Clone(),
           scaledTree);
@@ -323 +343 @@
         Results.Add(new Result(BestSolutionQualityParameterName, new DoubleValue()));
         Results.Add(new Result(CurrentBestValidationQualityParameterName, new DoubleValue()));
+        Results.Add(new Result("Best solution (training)", bestTrainingSolution));
       }
       Results[BestSolutionQualityParameterName].Value = new DoubleValue(BestSolutionQualityParameter.ActualValue.Value);
       Results[CurrentBestValidationQualityParameterName].Value = new DoubleValue(bestQuality);
+      Results["Best solution (training)"].Value = bestTrainingSolution;
 
       DataTable validationValues = (DataTable)Results[BestSolutionQualityValuesParameterName].Value;
@@ -332 +354 @@
 
       BestSolutionQualityValuesParameter.ActualValue = validationValues;
-      
+
       return base.Apply();
+    }
+
+    private SymbolicExpressionTree GetScaledTree(SymbolicExpressionTree tree) {
+      // calculate scaling parameters and only for the best tree using the full training set
+      double alpha, beta;
+      int trainingStart = ProblemData.TrainingSamplesStart.Value;
+      int trainingEnd = ProblemData.TrainingSamplesEnd.Value;
+      IEnumerable<int> trainingRows = Enumerable.Range(trainingStart, trainingEnd - trainingStart);
+      IEnumerable<double> originalValues = ProblemData.Dataset.GetEnumeratedVariableValues(ProblemData.TargetVariable.Value, trainingRows);
+      IEnumerable<double> estimatedValues = SymbolicExpressionTreeInterpreter.GetSymbolicExpressionTreeValues(tree, ProblemData.Dataset, trainingRows);
+
+      SymbolicRegressionScaledMeanSquaredErrorEvaluator.CalculateScalingParameters(originalValues, estimatedValues, out beta, out alpha);
+
+      // scale tree for solution
+      return SymbolicRegressionSolutionLinearScaler.Scale(tree, alpha, beta);
     }
 

branches/DataAnalysis/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/OverfittingAnalyzer.cs

@@ -23 +23 @@
 using System.Linq;
 using HeuristicLab.Analysis;
+using HeuristicLab.Common;
 using HeuristicLab.Core;
 using HeuristicLab.Data;
@@ -91 +92 @@
       get { return (ILookupParameter<PercentValue>)Parameters["RelativeValidationQuality"]; }
     }
+    //public IValueLookupParameter<PercentValue> RelativeValidationQualityLowerLimitParameter {
+    //  get { return (IValueLookupParameter<PercentValue>)Parameters["RelativeValidationQualityLowerLimit"]; }
+    //}
+    //public IValueLookupParameter<PercentValue> RelativeValidationQualityUpperLimitParameter {
+    //  get { return (IValueLookupParameter<PercentValue>)Parameters["RelativeValidationQualityUpperLimit"]; }
+    //}
     public ILookupParameter<DoubleValue> TrainingValidationQualityCorrelationParameter {
       get { return (ILookupParameter<DoubleValue>)Parameters["TrainingValidationCorrelation"]; }
@@ -102 +109 @@
     public ILookupParameter<ResultCollection> ResultsParameter {
       get { return (ILookupParameter<ResultCollection>)Parameters["Results"]; }
+    }
+    public ILookupParameter<DoubleValue> InitialTrainingQualityParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters["InitialTrainingQuality"]; }
     }
     #endregion
@@ -156 +166 @@
       Parameters.Add(new ValueLookupParameter<DoubleValue>(LowerEstimationLimitParameterName, "The lower estimation limit that was set for the evaluation of the symbolic expression trees."));
       Parameters.Add(new LookupParameter<PercentValue>("RelativeValidationQuality"));
+      //Parameters.Add(new ValueLookupParameter<PercentValue>("RelativeValidationQualityUpperLimit", new PercentValue(0.05)));
+      //Parameters.Add(new ValueLookupParameter<PercentValue>("RelativeValidationQualityLowerLimit", new PercentValue(-0.05)));
       Parameters.Add(new LookupParameter<DoubleValue>("TrainingValidationCorrelation"));
       Parameters.Add(new ValueLookupParameter<DoubleValue>("CorrelationLimit", new DoubleValue(0.65)));
       Parameters.Add(new LookupParameter<BoolValue>("Overfitting"));
       Parameters.Add(new LookupParameter<ResultCollection>("Results"));
+      Parameters.Add(new LookupParameter<DoubleValue>("InitialTrainingQuality"));
     }
 
@@ -167 +180 @@
     [StorableHook(HookType.AfterDeserialization)]
     private void AfterDeserialization() {
+      if (!Parameters.ContainsKey("InitialTrainingQuality")) {
+        Parameters.Add(new LookupParameter<DoubleValue>("InitialTrainingQuality"));
+      }
+      //if (!Parameters.ContainsKey("RelativeValidationQualityUpperLimit")) {
+      //  Parameters.Add(new ValueLookupParameter<PercentValue>("RelativeValidationQualityUpperLimit", new PercentValue(0.05)));
+      //}
+      //if (!Parameters.ContainsKey("RelativeValidationQualityLowerLimit")) {
+      //  Parameters.Add(new ValueLookupParameter<PercentValue>("RelativeValidationQualityLowerLimit", new PercentValue(-0.05)));
+      //}
     }
 
@@ -205 +227 @@
       //if (RelativeValidationQualityParameter.ActualValue == null) {
       // first call initialize the relative quality using the difference between average training and validation quality
-      double avgTrainingQuality = qualities.Average(x => x.Value);
-      double avgValidationQuality = validationQualities.Average();
+      double avgTrainingQuality = qualities.Select(x => x.Value).Median();
+      double avgValidationQuality = validationQualities.Median();
 
       if (Maximization.Value)
@@ -215 +237 @@
       //}
 
-      double[] validationArr = validationQualities.ToArray();
-      double[] trainingArr = qualities.Select(x => x.Value).ToArray();
-      double r = alglib.correlation.spearmanrankcorrelation(trainingArr, validationArr, trainingArr.Length);
+      // cut away 0.0 values to make the correlation stronger
+      // necessary because R² values of 0.0 are strong outliers
+      //int percentile = (int)Math.Round(0.1 * validationQualities.Count);
+      //double validationCutOffValue = validationQualities.OrderBy(x => x).ElementAt(percentile);
+      //double trainingCutOffValue = qualities.Select(x => x.Value).OrderBy(x => x).ElementAt(percentile);
+      double validationCutOffValue = 0.05;
+      double trainingCutOffValue = validationCutOffValue;
+
+      double[] validationArr = new double[validationQualities.Count];
+      double[] trainingArr = new double[validationQualities.Count];
+      int arrIndex = 0;
+      for (int i = 0; i < validationQualities.Count; i++) {
+        if (validationQualities[i] > validationCutOffValue &&
+            qualities[i].Value > trainingCutOffValue) {
+          validationArr[arrIndex] = validationQualities[i];
+          trainingArr[arrIndex] = qualities[i].Value;
+          arrIndex++;
+        }
+      }
+      double r = alglib.correlation.spearmanrankcorrelation(trainingArr, validationArr, arrIndex);
       TrainingValidationQualityCorrelationParameter.ActualValue = new DoubleValue(r);
-      OverfittingParameter.ActualValue = new BoolValue(RelativeValidationQualityParameter.ActualValue.Value < 0 && r < CorrelationLimitParameter.ActualValue.Value);
+      if (InitialTrainingQualityParameter.ActualValue == null)
+        InitialTrainingQualityParameter.ActualValue = new DoubleValue(avgValidationQuality);
+      bool overfitting =
+        avgTrainingQuality > InitialTrainingQualityParameter.ActualValue.Value &&  // better on training than in initial generation
+        r < CorrelationLimitParameter.ActualValue.Value;  // low correlation between training and validation quality
+
+      //// if validation quality is within a certain margin of percentage deviation (default -5% .. 5%) then there is no overfitting
+      //// correlation is also bad when underfitting but validation quality cannot be a lot larger than training quality if overfitting
+      //(RelativeValidationQualityParameter.ActualValue.Value > RelativeValidationQualityUpperLimitParameter.ActualValue.Value || // better on training than on validation
+      // RelativeValidationQualityParameter.ActualValue.Value < RelativeValidationQualityLowerLimitParameter.ActualValue.Value); // better on training than on validation
+
+      OverfittingParameter.ActualValue = new BoolValue(overfitting);
       return base.Apply();
     }

branches/DataAnalysis/HeuristicLab.Problems.DataAnalysis/3.3/Operators/CovariantParsimonyPressure.cs

@@ -89 +89 @@
     public ILookupParameter<DataTable> QualitiesParameter {
       get { return (ILookupParameter<DataTable>)Parameters["Qualities"]; }
+    }
+    public IValueLookupParameter<DoubleValue> MinAverageSizeParameter {
+      get { return (IValueLookupParameter<DoubleValue>)Parameters["MinAverageSize"]; }
     }
 
@@ -111 +114 @@
       Parameters.Add(new LookupParameter<DoubleValue>("Correlation(Length, AdjustedFitness)"));
       Parameters.Add(new LookupParameter<DoubleValue>("Correlation(Fitness, AdjustedFitness)"));
+      Parameters.Add(new ValueLookupParameter<DoubleValue>("MinAverageSize", new DoubleValue(15)));
     }
 
@@ -151 +155 @@
       if (!Parameters.ContainsKey("ComplexityAdaption")) {
         Parameters.Add(new ValueLookupParameter<PercentValue>("ComplexityAdaption", new PercentValue(-5)));
+      }
+      if (!Parameters.ContainsKey("MinAverageSize")) {
+        Parameters.Add(new ValueLookupParameter<DoubleValue>("MinAverageSize", new DoubleValue(15)));
       }
     }
@@ -190 +197 @@
         else sizeAdaption = Math.Ceiling(sizeAdaption);
         double g = lengthMeanCalculator.Mean + sizeAdaption;
+        if (g < MinAverageSizeParameter.ActualValue.Value)
+          g = MinAverageSizeParameter.ActualValue.Value;
 
         //            cov(l, f) - (g(t+1) - mu(t)) avgF