Changeset 4297 for branches/DataAnalysis/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/OverfittingAnalyzer.cs

Timestamp:

08/23/10 18:54:35 (14 years ago)

Author:

gkronber

Message:

Added output parameter for validation quality to validation analyzer, added input parameter for validation quality to overfitting analyzer, and fixed bugs in pruning operator. #1142

File:

• branches/DataAnalysis/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/OverfittingAnalyzer.cs (modified) (7 diffs)

branches/DataAnalysis/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/OverfittingAnalyzer.cs

@@ -61 +61 @@
       get { return (ScopeTreeLookupParameter<DoubleValue>)Parameters["Quality"]; }
     }
+    public ScopeTreeLookupParameter<DoubleValue> ValidationQualityParameter {
+      get { return (ScopeTreeLookupParameter<DoubleValue>)Parameters["ValidationQuality"]; }
+    }
     public IValueLookupParameter<ISymbolicExpressionTreeInterpreter> SymbolicExpressionTreeInterpreterParameter {
       get { return (IValueLookupParameter<ISymbolicExpressionTreeInterpreter>)Parameters[SymbolicExpressionTreeInterpreterParameterName]; }
@@ -163 +166 @@
       Parameters.Add(new ScopeTreeLookupParameter<SymbolicExpressionTree>(SymbolicExpressionTreeParameterName, "The symbolic expression trees to analyze."));
       Parameters.Add(new ScopeTreeLookupParameter<DoubleValue>("Quality"));
+      Parameters.Add(new ScopeTreeLookupParameter<DoubleValue>("ValidationQuality"));
       Parameters.Add(new LookupParameter<BoolValue>(MaximizationParameterName, "The direction of optimization."));
       Parameters.Add(new ValueLookupParameter<ISymbolicExpressionTreeInterpreter>(SymbolicExpressionTreeInterpreterParameterName, "The interpreter that should be used for the analysis of symbolic expression trees."));
@@ -180 +184 @@
       Parameters.Add(new LookupParameter<DoubleValue>("InitialTrainingQuality"));
       Parameters.Add(new LookupParameter<DoubleMatrix>("TrainingAndValidationQualities"));
-      Parameters.Add(new ValueLookupParameter<DoubleValue>("Percentile", new DoubleValue(0.1)));
+      Parameters.Add(new ValueLookupParameter<DoubleValue>("Percentile", new DoubleValue(1)));
 
     }
@@ -202 +206 @@
       }
       if (!Parameters.ContainsKey("Percentile")) {
-        Parameters.Add(new ValueLookupParameter<DoubleValue>("Percentile", new DoubleValue(0.1)));
+        Parameters.Add(new ValueLookupParameter<DoubleValue>("Percentile", new DoubleValue(1)));
+      }
+      if (!Parameters.ContainsKey("ValidationQuality")) {
+        Parameters.Add(new ScopeTreeLookupParameter<DoubleValue>("ValidationQuality"));
       }
     }
@@ -209 +216 @@
       var trees = SymbolicExpressionTree;
       ItemArray<DoubleValue> qualities = QualityParameter.ActualValue;
+      ItemArray<DoubleValue> validationQualities = ValidationQualityParameter.ActualValue;
 
       string targetVariable = ProblemData.TargetVariable.Value;
@@ -226 +234 @@
       //SymbolicExpressionTree bestTree = null;
 
-      List<double> validationQualities = new List<double>();
-      foreach (var tree in trees) {
-        double quality = Evaluator.Evaluate(SymbolicExpressionTreeInterpreter, tree,
-          lowerEstimationLimit, upperEstimationLimit,
-          ProblemData.Dataset, targetVariable,
-         rows);
-        validationQualities.Add(quality);
-        //if ((Maximization.Value && quality > bestQuality) ||
-        //    (!Maximization.Value && quality < bestQuality)) {
-        //  bestQuality = quality;
-        //  bestTree = tree;
-        //}
-      }
+      //List<double> validationQualities = new List<double>();
+      //foreach (var tree in trees) {
+      //  double quality = Evaluator.Evaluate(SymbolicExpressionTreeInterpreter, tree,
+      //    lowerEstimationLimit, upperEstimationLimit,
+      //    ProblemData.Dataset, targetVariable,
+      //   rows);
+      //  validationQualities.Add(quality);
+      //  //if ((Maximization.Value && quality > bestQuality) ||
+      //  //    (!Maximization.Value && quality < bestQuality)) {
+      //  //  bestQuality = quality;
+      //  //  bestTree = tree;
+      //  //}
+      //}
 
       //if (RelativeValidationQualityParameter.ActualValue == null) {
       // first call initialize the relative quality using the difference between average training and validation quality
       double avgTrainingQuality = qualities.Select(x => x.Value).Median();
-      double avgValidationQuality = validationQualities.Median();
+      double avgValidationQuality = validationQualities.Select(x => x.Value).Median();
 
       if (Maximization.Value)
@@ -254 +262 @@
       // best first (only for maximization
       var orderedDistinctPairs = (from index in Enumerable.Range(0, qualities.Length)
-                                  select new { Training = qualities[index].Value, Validation = validationQualities[index] })
-                                 .Distinct()
+                                  select new { Training = qualities[index].Value, Validation = validationQualities[index].Value })
                                  .OrderBy(x => -x.Training)
                                  .ToList();