Changeset 7721

Timestamp:

04/11/12 07:37:02 (12 years ago)

Author:

gkronber

Message:

#1817 added parameter to symbolic data analysis validation analyzers to determine the percentage of best solutions that should be evaluated on the validation set

Location:

trunk/sources/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Analyzers

Files:

• SymbolicDataAnalysisSingleObjectiveValidationAnalyzer.cs (modified) (3 diffs)
• SymbolicDataAnalysisSingleObjectiveValidationBestSolutionAnalyzer.cs (modified) (3 diffs)

trunk/sources/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Analyzers/SymbolicDataAnalysisSingleObjectiveValidationAnalyzer.cs

@@ -45 +45 @@
     private const string ValidationPartitionParameterName = "ValidationPartition";
     private const string RelativeNumberOfEvaluatedSamplesParameterName = "RelativeNumberOfEvaluatedSamples";
+    private const string PercentageOfBestSolutionsParameterName = "PercentageOfBestSolutions";
 
     #region parameter properties
@@ -65 +66 @@
       get { return (IValueLookupParameter<PercentValue>)Parameters[RelativeNumberOfEvaluatedSamplesParameterName]; }
     }
+    public IValueLookupParameter<PercentValue> PercentageOfBestSolutionsParameter {
+      get { return (IValueLookupParameter<PercentValue>)Parameters[PercentageOfBestSolutionsParameterName]; }
+    }
     #endregion
 
@@ -80 +84 @@
       Parameters.Add(new ValueLookupParameter<IntRange>(ValidationPartitionParameterName, "Thes validation partition."));
       Parameters.Add(new ValueLookupParameter<PercentValue>(RelativeNumberOfEvaluatedSamplesParameterName, "The relative number of samples of the dataset partition, which should be randomly chosen for evaluation between the start and end index."));
+      Parameters.Add(new ValueLookupParameter<PercentValue>(PercentageOfBestSolutionsParameterName,
+                                                            "The percentage of the top solutions which should be analyzed.", new PercentValue(0.1)));
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      if (!Parameters.ContainsKey(PercentageOfBestSolutionsParameterName))
+        Parameters.Add(new ValueLookupParameter<PercentValue>(PercentageOfBestSolutionsParameterName,
+                                                               "The percentage of the top solutions which should be analyzed.", new PercentValue(1)));
     }
 

trunk/sources/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Analyzers/SymbolicDataAnalysisSingleObjectiveValidationBestSolutionAnalyzer.cs

@@ -20 +20 @@
 #endregion
 
+using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -72 +73 @@
 
     public override IOperation Apply() {
-      #region find best tree
-      double bestQuality = Maximization.Value ? double.NegativeInfinity : double.PositiveInfinity;
-      ISymbolicExpressionTree bestTree = null;
-      ISymbolicExpressionTree[] tree = SymbolicExpressionTree.ToArray();
-      var evaluator = EvaluatorParameter.ActualValue;
-      var problemData = ProblemDataParameter.ActualValue;
       IEnumerable<int> rows = GenerateRowsToEvaluate();
       if (!rows.Any()) return base.Apply();
 
+      #region find best tree
+      var evaluator = EvaluatorParameter.ActualValue;
+      var problemData = ProblemDataParameter.ActualValue;
+      double bestValidationQuality = Maximization.Value ? double.NegativeInfinity : double.PositiveInfinity;
+      ISymbolicExpressionTree bestTree = null;
+      ISymbolicExpressionTree[] tree = SymbolicExpressionTree.ToArray();
+
+      // sort is ascending and we take the first n% => order so that best solutions are smallest
+      // sort order is determined by maximization parameter
+      double[] trainingQuality;
+      if (Maximization.Value) {
+        // largest values must be sorted first
+        trainingQuality = Quality.Select(x => -x.Value).ToArray();
+      } else {
+        // smallest values must be sorted first
+        trainingQuality = Quality.Select(x => x.Value).ToArray();
+      }
+
+      // sort trees by training qualities
+      Array.Sort(trainingQuality, tree);
+
+      // number of best training solutions to validate (at least 1)
+      int topN = (int)Math.Max(tree.Length * PercentageOfBestSolutionsParameter.ActualValue.Value, 1);
+
       IExecutionContext childContext = (IExecutionContext)ExecutionContext.CreateChildOperation(evaluator);
+      // evaluate best n training trees on validiation set
       var quality = tree
+        .Take(topN)
         .AsParallel()
         .Select(t => evaluator.Evaluate(childContext, t, problemData, rows))
         .ToArray();
 
-      for (int i = 0; i < tree.Length; i++) {
-        if (IsBetter(quality[i], bestQuality, Maximization.Value)) {
-          bestQuality = quality[i];
+      for (int i = 0; i < quality.Length; i++) {
+        if (IsBetter(quality[i], bestValidationQuality, Maximization.Value)) {
+          bestValidationQuality = quality[i];
           bestTree = tree[i];
         }
@@ -97 +118 @@
       var results = ResultCollection;
       if (ValidationBestSolutionQuality == null ||
-        IsBetter(bestQuality, ValidationBestSolutionQuality.Value, Maximization.Value)) {
-        ValidationBestSolution = CreateSolution(bestTree, bestQuality);
-        ValidationBestSolutionQuality = new DoubleValue(bestQuality);
+        IsBetter(bestValidationQuality, ValidationBestSolutionQuality.Value, Maximization.Value)) {
+        ValidationBestSolution = CreateSolution(bestTree, bestValidationQuality);
+        ValidationBestSolutionQuality = new DoubleValue(bestValidationQuality);
 
         if (!results.ContainsKey(ValidationBestSolutionParameter.Name)) {