Changeset 4191

Timestamp:

08/11/10 12:00:53 (14 years ago)

Author:

gkronber

Message:

Changed validation best solution analyzer and tournament pruning operator to use the evaluator specified in the problem parameters. #1117

Location:

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers

Files:

• FixedValidationBestScaledSymbolicRegressionSolutionAnalyzer.cs (modified) (9 diffs)
• SymbolicRegressionTournamentPruning.cs (modified) (10 diffs)

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/FixedValidationBestScaledSymbolicRegressionSolutionAnalyzer.cs

@@ -46 +46 @@
     private const string ValidationSamplesStartParameterName = "SamplesStart";
     private const string ValidationSamplesEndParameterName = "SamplesEnd";
-    private const string QualityParameterName = "Quality";
+    // private const string QualityParameterName = "Quality";
     private const string UpperEstimationLimitParameterName = "UpperEstimationLimit";
     private const string LowerEstimationLimitParameterName = "LowerEstimationLimit";
+    private const string EvaluatorParameterName = "Evaluator";
+    private const string MaximizationParameterName = "Maximization";
     private const string BestSolutionParameterName = "Best solution (validation)";
     private const string BestSolutionQualityParameterName = "Best solution quality (validation)";
@@ -109 +111 @@
       get { return (IValueLookupParameter<ISymbolicExpressionTreeInterpreter>)Parameters[SymbolicExpressionTreeInterpreterParameterName]; }
     }
+    public ILookupParameter<ISymbolicRegressionEvaluator> EvaluatorParameter {
+      get { return (ILookupParameter<ISymbolicRegressionEvaluator>)Parameters[EvaluatorParameterName]; }
+    }
+    public ILookupParameter<BoolValue> MaximizationParameter {
+      get { return (ILookupParameter<BoolValue>)Parameters[MaximizationParameterName]; }
+    }
     public IValueLookupParameter<DataAnalysisProblemData> ProblemDataParameter {
       get { return (IValueLookupParameter<DataAnalysisProblemData>)Parameters[ProblemDataParameterName]; }
@@ -158 +166 @@
       get { return SymbolicExpressionTreeInterpreterParameter.ActualValue; }
     }
+    public ISymbolicRegressionEvaluator Evaluator {
+      get { return EvaluatorParameter.ActualValue; }
+    }
+    public BoolValue Maximization {
+      get { return MaximizationParameter.ActualValue; }
+    }
     public DataAnalysisProblemData ProblemData {
       get { return ProblemDataParameter.ActualValue; }
@@ -185 +199 @@
     public IntValue Generations {
       get { return GenerationsParameter.ActualValue; }
+    }
+    public DoubleValue BestSolutionQuality {
+      get { return BestSolutionQualityParameter.ActualValue; }
     }
 
@@ -192 +209 @@
       : base() {
       Parameters.Add(new LookupParameter<IRandom>(RandomParameterName, "The random generator to use."));
+      Parameters.Add(new LookupParameter<ISymbolicRegressionEvaluator>(EvaluatorParameterName, "The evaluator which should be used to evaluate the solution on the validation set."));
       Parameters.Add(new ScopeTreeLookupParameter<SymbolicExpressionTree>(SymbolicExpressionTreeParameterName, "The symbolic expression trees to analyze."));
-      Parameters.Add(new ScopeTreeLookupParameter<DoubleValue>(QualityParameterName, "The quality of the symbolic expression trees to analyze."));
+      Parameters.Add(new LookupParameter<BoolValue>(MaximizationParameterName, "The direction of optimization."));
       Parameters.Add(new ValueLookupParameter<ISymbolicExpressionTreeInterpreter>(SymbolicExpressionTreeInterpreterParameterName, "The interpreter that should be used for the analysis of symbolic expression trees."));
       Parameters.Add(new ValueLookupParameter<DataAnalysisProblemData>(ProblemDataParameterName, "The problem data for which the symbolic expression tree is a solution."));
@@ -212 +230 @@
     private FixedValidationBestScaledSymbolicRegressionSolutionAnalyzer(bool deserializing) : base() { }
 
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      #region compatibility remove before releasing 3.3.1
+      if (!Parameters.ContainsKey(EvaluatorParameterName)) {
+        Parameters.Add(new LookupParameter<ISymbolicRegressionEvaluator>(EvaluatorParameterName, "The evaluator which should be used to evaluate the solution on the validation set."));
+      }
+      if (!Parameters.ContainsKey(MaximizationParameterName)) {
+        Parameters.Add(new LookupParameter<BoolValue>(MaximizationParameterName, "The direction of optimization."));
+      }
+      #endregion
+    }
+
     public override IOperation Apply() {
       var trees = SymbolicExpressionTree;
@@ -228 +258 @@
       double lowerEstimationLimit = LowerEstimationLimit != null ? LowerEstimationLimit.Value : double.NegativeInfinity;
 
-      double bestValidationRSquared = -1.0;
+      double bestQuality = Maximization.Value ? double.NegativeInfinity : double.PositiveInfinity;
       SymbolicExpressionTree bestTree = null;
 
       foreach (var tree in trees) {
-        double validationRSquared = SymbolicRegressionPearsonsRSquaredEvaluator.Calculate(SymbolicExpressionTreeInterpreter, tree,
+        double quality = Evaluator.Evaluate(SymbolicExpressionTreeInterpreter, tree,
           lowerEstimationLimit, upperEstimationLimit,
           ProblemData.Dataset, targetVariable,
          rows);
 
-        if (validationRSquared > bestValidationRSquared) {
-          bestValidationRSquared = validationRSquared;
+        if ((Maximization.Value && quality > bestQuality) ||
+            (!Maximization.Value && quality < bestQuality)) {
+          bestQuality = quality;
           bestTree = tree;
         }
       }
 
-
       // if the best validation tree is better than the current best solution => update
-      if (BestSolutionQualityParameter.ActualValue == null || BestSolutionQualityParameter.ActualValue.Value < bestValidationRSquared) {
-        // calculate scaling parameters and validation MSE only for the best tree
-        // scale tree for solution
+      bool newBest =
+        BestSolutionQuality == null ||
+        (Maximization.Value && bestQuality > BestSolutionQuality.Value) ||
+        (!Maximization.Value && bestQuality < BestSolutionQuality.Value);
+      if (newBest) {
+        // calculate scaling parameters and only for the best tree using the full training set
         double alpha, beta;
-        double validationMSE = SymbolicRegressionScaledMeanSquaredErrorEvaluator.Calculate(SymbolicExpressionTreeInterpreter, bestTree,
+        int trainingStart = ProblemData.TrainingSamplesStart.Value;
+        int trainingEnd = ProblemData.TrainingSamplesEnd.Value;
+        IEnumerable<int> trainingRows = Enumerable.Range(trainingStart, trainingEnd - trainingStart);
+        SymbolicRegressionScaledMeanSquaredErrorEvaluator.Calculate(SymbolicExpressionTreeInterpreter, bestTree,
           lowerEstimationLimit, upperEstimationLimit,
           ProblemData.Dataset, targetVariable,
-          rows, out beta, out alpha);
-
+          trainingRows, out beta, out alpha);
+
+        // scale tree for solution
         var scaledTree = SymbolicRegressionSolutionLinearScaler.Scale(bestTree, alpha, beta);
         var model = new SymbolicRegressionModel((ISymbolicExpressionTreeInterpreter)SymbolicExpressionTreeInterpreter.Clone(),
@@ -262 +299 @@
 
         BestSolutionParameter.ActualValue = solution;
-        BestSolutionQualityParameter.ActualValue = new DoubleValue(bestValidationRSquared);
+        BestSolutionQualityParameter.ActualValue = new DoubleValue(bestQuality);
 
         BestSymbolicRegressionSolutionAnalyzer.UpdateBestSolutionResults(solution, ProblemData, Results, Generations, VariableFrequencies);
       }
+
 
       if (!Results.ContainsKey(BestSolutionQualityValuesParameterName)) {
@@ -273 +311 @@
       }
       Results[BestSolutionQualityParameterName].Value = new DoubleValue(BestSolutionQualityParameter.ActualValue.Value);
-      Results[CurrentBestValidationQualityParameterName].Value = new DoubleValue(bestValidationRSquared);
+      Results[CurrentBestValidationQualityParameterName].Value = new DoubleValue(bestQuality);
 
       DataTable validationValues = (DataTable)Results[BestSolutionQualityValuesParameterName].Value;
       AddValue(validationValues, BestSolutionQualityParameter.ActualValue.Value, BestSolutionQualityParameterName, BestSolutionQualityParameterName);
-      AddValue(validationValues, bestValidationRSquared, CurrentBestValidationQualityParameterName, CurrentBestValidationQualityParameterName);
+      AddValue(validationValues, bestQuality, CurrentBestValidationQualityParameterName, CurrentBestValidationQualityParameterName);
       return base.Apply();
     }

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/SymbolicRegressionTournamentPruning.cs

@@ -30 +30 @@
 using HeuristicLab.Problems.DataAnalysis.Symbolic;
 using HeuristicLab.Problems.DataAnalysis.Symbolic.Symbols;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.Problems.DataAnalysis.Regression.Symbolic.Analyzers {
@@ -38 +39 @@
     private const string SamplesStartParameterName = "SamplesStart";
     private const string SamplesEndParameterName = "SamplesEnd";
+    private const string EvaluatorParameterName = "Evaluator";
+    private const string MaximizationParameterName = "Maximization";
     private const string SymbolicExpressionTreeInterpreterParameterName = "SymbolicExpressionTreeInterpreter";
     private const string UpperEstimationLimitParameterName = "UpperEstimationLimit";
@@ -77 +80 @@
       get { return (IValueLookupParameter<IntValue>)Parameters[SamplesEndParameterName]; }
     }
+    public ILookupParameter<ISymbolicRegressionEvaluator> EvaluatorParameter {
+      get { return (ILookupParameter<ISymbolicRegressionEvaluator>)Parameters[EvaluatorParameterName]; }
+    }
+    public ILookupParameter<BoolValue> MaximizationParameter {
+      get { return (ILookupParameter<BoolValue>)Parameters[MaximizationParameterName]; }
+    }
     public IValueLookupParameter<DoubleValue> MaxPruningRatioParameter {
       get { return (IValueLookupParameter<DoubleValue>)Parameters[MaxPruningRatioParameterName]; }
@@ -133 +142 @@
       get { return SamplesEndParameter.ActualValue; }
     }
+    public ISymbolicRegressionEvaluator Evaluator {
+      get { return EvaluatorParameter.ActualValue; }
+    }
+    public BoolValue Maximization {
+      get { return MaximizationParameter.ActualValue; }
+    }
     public DoubleValue MaxPruningRatio {
       get { return MaxPruningRatioParameter.ActualValue; }
@@ -161 +176 @@
     }
     #endregion
+    protected SymbolicRegressionTournamentPruning(bool deserializing) : base(deserializing) { }
     public SymbolicRegressionTournamentPruning()
       : base() {
@@ -169 +185 @@
       Parameters.Add(new ValueLookupParameter<IntValue>(SamplesStartParameterName, "The first row index of the dataset partition to use for branch impact evaluation."));
       Parameters.Add(new ValueLookupParameter<IntValue>(SamplesEndParameterName, "The last row index of the dataset partition to use for branch impact evaluation."));
+      Parameters.Add(new LookupParameter<ISymbolicRegressionEvaluator>(EvaluatorParameterName, "The evaluator that should be used to determine which branches are not relevant."));
+      Parameters.Add(new LookupParameter<BoolValue>(MaximizationParameterName, "The direction of optimization."));
       Parameters.Add(new ValueLookupParameter<DoubleValue>(MaxPruningRatioParameterName, "The maximal relative size of the pruned branch.", new DoubleValue(0.5)));
       Parameters.Add(new ValueLookupParameter<IntValue>(TournamentSizeParameterName, "The number of branches to compare for pruning", new IntValue(10)));
@@ -181 +199 @@
       Parameters.Add(new LookupParameter<IntValue>(GenerationParameterName, "The current generation."));
       Parameters.Add(new LookupParameter<ResultCollection>(ResultsParameterName, "The results collection."));
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      #region compatibility remove before releasing 3.3.1
+      if (!Parameters.ContainsKey(EvaluatorParameterName)) {
+        Parameters.Add(new LookupParameter<ISymbolicRegressionEvaluator>(EvaluatorParameterName, "The evaluator which should be used to evaluate the solution on the validation set."));
+      }
+      if (!Parameters.ContainsKey(MaximizationParameterName)) {
+        Parameters.Add(new LookupParameter<BoolValue>(MaximizationParameterName, "The direction of optimization."));
+      }
+      #endregion
     }
 
@@ -198 +228 @@
           Prune(Random, tree, Iterations.Value, TournamentSize.Value,
             DataAnalysisProblemData, SamplesStart.Value, SamplesEnd.Value,
-            SymbolicExpressionTreeInterpreter,
+            SymbolicExpressionTreeInterpreter, Evaluator, Maximization.Value,
             LowerEstimationLimit.Value, UpperEstimationLimit.Value,
             MaxPruningRatio.Value, QualityGainWeight.Value);
@@ -208 +238 @@
     public static void Prune(IRandom random, SymbolicExpressionTree tree, int iterations, int tournamentSize,
       DataAnalysisProblemData problemData, int samplesStart, int samplesEnd,
-      ISymbolicExpressionTreeInterpreter interpreter,
+      ISymbolicExpressionTreeInterpreter interpreter, ISymbolicRegressionEvaluator evaluator, bool maximization,
       double lowerEstimationLimit, double upperEstimationLimit,
       double maxPruningRatio, double qualityGainWeight) {
       IEnumerable<int> rows = Enumerable.Range(samplesStart, samplesEnd - samplesStart);
       int originalSize = tree.Size;
-      double originalMse = SymbolicRegressionScaledMeanSquaredErrorEvaluator.Calculate(interpreter, tree,
-        lowerEstimationLimit, upperEstimationLimit, problemData.Dataset, problemData.TargetVariable.Value, Enumerable.Range(samplesStart, samplesEnd - samplesStart));
+      double originalQuality = evaluator.Evaluate(interpreter, tree,
+        lowerEstimationLimit, upperEstimationLimit, problemData.Dataset, problemData.TargetVariable.Value, rows);
 
       int minPrunedSize = (int)(originalSize * (1 - maxPruningRatio));
@@ -249 +279 @@
             selectedPrunePoint.Parent.InsertSubTree(selectedPrunePoint.SubTreeIndex, constNode);
 
-            double prunedMse = SymbolicRegressionScaledMeanSquaredErrorEvaluator.Calculate(interpreter, clonedTree,
+            double prunedQuality = evaluator.Evaluate(interpreter, clonedTree,
         lowerEstimationLimit, upperEstimationLimit, problemData.Dataset, problemData.TargetVariable.Value, Enumerable.Range(samplesStart, samplesEnd - samplesStart));
             double prunedSize = clonedTree.Size;
-            // MSE of the pruned tree is larger than the original tree in most cases
+            // deteriation in quality:
+            // exp: MSE : newMse < origMse (improvement) => prefer the larger improvement
+            //      MSE : newMse > origMse (deteriation) => prefer the smaller deteriation
+            //      MSE : minimize: newMse / origMse
+            //      R²  : newR² > origR²   (improvment) => prefer the larger improvment
+            //      R²  : newR² < origR²   (deteriation) => prefer smaller deteriation
+            //      R²  : minimize: origR² / newR²
+            double qualityDeteriation = maximization ? originalQuality / prunedQuality : prunedQuality / originalQuality;
             // size of the pruned tree is always smaller than the size of the original tree
             // same change in quality => prefer pruning operation that removes a larger tree
-            double gain = ((prunedMse / originalMse) * qualityGainWeight) /
+            double gain = (qualityDeteriation * qualityGainWeight) /
                            (originalSize / prunedSize);
             if (gain < bestGain) {