Changeset 5212 for branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/ParameterConfigurationEvaluator.cs

Timestamp:

01/05/11 02:03:20 (14 years ago)

Author:

cneumuel

Message:

#1215

• added normalization for quality values of individuals

File:

• branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/ParameterConfigurationEvaluator.cs (modified) (7 diffs)

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/ParameterConfigurationEvaluator.cs

@@ -40 +40 @@
     }
 
+    public LookupParameter<DoubleArray> ProblemQualityMediansParameter {
+      get { return (LookupParameter<DoubleArray>)Parameters["ProblemQualityMedians"]; }
+    }
+
     public IntValue Repetitions {
       get { return RepetitionsParameter.ActualValue; }
@@ -51 +55 @@
       Parameters.Add(new LookupParameter<ParameterConfigurationTree>("ParameterConfigurationTree", "Missing description."));
       Parameters.Add(new LookupParameter<IntValue>(MetaOptimizationProblem.RepetitionsParameterName, "Number of evaluations on one problem."));
+      Parameters.Add(new LookupParameter<DoubleArray>("ProblemQualityMedians", ""));
     }
 
@@ -66 +71 @@
       IItemList<ISingleObjectiveProblem> problems = ProblemsParameter.ActualValue;
 
+      double[] medians;
+      if (ProblemQualityMediansParameter.ActualValue == null) {
+        // this is generation zero. no reference qualities for scaling have been calculated yet. calculate a random individuum to get reference values
+        medians = new double[problems.Count]; // todo
+        for (int i = 0; i < medians.Length; i++) {
+          medians[i] = 10;
+        }
+      } else {
+        medians = ProblemQualityMediansParameter.ActualValue.ToArray();
+      }
+
       // set parameters
       ParameterConfigurationParameter.ActualValue.Parameterize(algorithm);
       algorithm.StoreAlgorithmInEachRun = false;
 
-      List<double> qualities = new List<double>();
-      List<TimeSpan> executionTimes = new List<TimeSpan>();
+      List<List<double>> qualities = new List<List<double>>();
+      List<List<TimeSpan>> executionTimes = new List<List<TimeSpan>>();
       algorithm.Engine = new SequentialEngine.SequentialEngine();
       algorithm.Prepare(true);
@@ -77 +93 @@
       foreach (ISingleObjectiveProblem problem in problems) {
         algorithm.Problem = (IProblem)problem.Clone();
+        var problemQualities = new List<double>();
+        var problemExecutionTimes = new List<TimeSpan>();
 
         for (int i = 0; i < Repetitions.Value; i++) {
@@ -86 +104 @@
           if (algorithm.ExecutionState == ExecutionState.Paused) {
             // this parametercombination was bad. set penalty for this solution
-            qualities.Add(PenaltyQuality); // todo: respect Maximization; problem: this messes up average value; solution: use currently worst quality*2
-            executionTimes.Add(algorithm.ExecutionTime);
+            problemQualities.Add(PenaltyQuality); // todo: respect Maximization; problem: this messes up average value; solution: use currently worst quality*2
+            problemExecutionTimes.Add(algorithm.ExecutionTime);
           } else {
-            qualities.Add(((DoubleValue)algorithm.Results["BestQuality"].Value).Value);
-            executionTimes.Add(algorithm.ExecutionTime);
+            problemQualities.Add(((DoubleValue)algorithm.Results["BestQuality"].Value).Value);
+            problemExecutionTimes.Add(algorithm.ExecutionTime);
 
             // parameters will be stored in ParameterConfigurationTree anyway. they would be redundant in runs
@@ -98 +116 @@
           }
         }
+        qualities.Add(problemQualities);
+        executionTimes.Add(problemExecutionTimes);
       }
       algorithm.Prepare();
 
-      qualities = qualities.OrderBy(x => x).ToList();  // todo: respect Maximization:true/false
-
-      ParameterConfigurationParameter.ActualValue.AverageExecutionTime = new TimeSpanValue(TimeSpan.FromMilliseconds(executionTimes.Average(t => t.TotalMilliseconds)));
+      ParameterConfigurationParameter.ActualValue.AverageExecutionTimes = new ItemList<TimeSpanValue>(executionTimes.Select(t => new TimeSpanValue(TimeSpan.FromMilliseconds(t.Average(ts => ts.TotalMilliseconds)))));
       ParameterConfigurationParameter.ActualValue.Repetitions = (IntValue)Repetitions.Clone();
-      ParameterConfigurationParameter.ActualValue.BestQuality = new DoubleValue(qualities.First());
-      ParameterConfigurationParameter.ActualValue.AverageQuality = new DoubleValue(qualities.Average());
-      ParameterConfigurationParameter.ActualValue.WorstQuality = new DoubleValue(qualities.Last());
-      ParameterConfigurationParameter.ActualValue.QualityVariance = new DoubleValue(qualities.Variance());
-      ParameterConfigurationParameter.ActualValue.QualityStandardDeviation = new DoubleValue(qualities.StandardDeviation());
+      ParameterConfigurationParameter.ActualValue.BestQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray()); // todo: respect Maximization:true/false
+      ParameterConfigurationParameter.ActualValue.AverageQualities = new DoubleArray(qualities.Select(q => q.Average()).ToArray());
+      ParameterConfigurationParameter.ActualValue.WorstQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray()); // todo: respect Maximization:true/false
+      ParameterConfigurationParameter.ActualValue.QualityVariances = new DoubleArray(qualities.Select(q => q.Variance()).ToArray());
+      ParameterConfigurationParameter.ActualValue.QualityStandardDeviations = new DoubleArray(qualities.Select(q => q.StandardDeviation()).ToArray());
       ParameterConfigurationParameter.ActualValue.Runs = (RunCollection)algorithm.Runs.Clone();
 
-      double quality = ParameterConfigurationParameter.ActualValue.AverageQuality.Value; // todo: also include other measures (executiontime, variance)
-      this.QualityParameter.ActualValue = new DoubleValue(quality);
+      // normalize qualities
+      double[] qualitiesNormalized = new double[problems.Count];
+      for (int i = 0; i < problems.Count; i++) {
+        qualitiesNormalized[i] = qualities[i].Average() / medians[i];
+      }
+      ParameterConfigurationParameter.ActualValue.QualitiesNormalized = new DoubleArray(qualitiesNormalized);
+      ParameterConfigurationParameter.ActualValue.AverageQualityNormalized = new DoubleValue(qualitiesNormalized.Average());
+
+      this.QualityParameter.ActualValue = ParameterConfigurationParameter.ActualValue.AverageQualityNormalized;
 
       return base.Apply();
@@ -129 +154 @@
   }
 
+  /// <summary>
+  /// Can execute an algorithm synchronously
+  /// </summary>
   public class AlgorithmExecutor {
     private EngineAlgorithm algorithm;