Changeset 5303 for branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/ParameterConfigurationEvaluator.cs

Timestamp:

01/17/11 01:04:30 (14 years ago)

Author:

cneumuel

Message:

#1215

• manipulators for one and all parameters
• SolutionCache to avoid multiple evaluations of equal solutions
• RunsAnalyzer which stores all base level runs
• ItemDictionaryView for runs

File:

• branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/ParameterConfigurationEvaluator.cs (modified) (4 diffs)

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/ParameterConfigurationEvaluator.cs

@@ -43 +43 @@
       get { return (LookupParameter<DoubleArray>)Parameters["ProblemQualityReferences"]; }
     }
-
+    public LookupParameter<IntValue> GenerationsParameter {
+      get { return (LookupParameter<IntValue>)Parameters["Generations"]; }
+    }
+    public LookupParameter<ResultCollection> ResultsParameter {
+      get { return (LookupParameter<ResultCollection>)Parameters["Results"]; }
+    }
+    
     public IntValue Repetitions {
       get { return RepetitionsParameter.ActualValue; }
@@ -56 +62 @@
       Parameters.Add(new LookupParameter<IntValue>(MetaOptimizationProblem.RepetitionsParameterName, "Number of evaluations on one problem."));
       Parameters.Add(new LookupParameter<DoubleArray>("ProblemQualityReferences", ""));
+      Parameters.Add(new LookupParameter<IntValue>("Generations", ""));
+      Parameters.Add(new LookupParameter<ResultCollection>("Results", ""));
     }
 
@@ -68 +76 @@
 
     public override IOperation Apply() {
-      EngineAlgorithm algorithm = (EngineAlgorithm)AlgorithmParameter.ActualValue.Clone();
+      ParameterConfigurationTree parameterConfiguration = ParameterConfigurationParameter.ActualValue;
+      EngineAlgorithm algorithm = (EngineAlgorithm)AlgorithmParameter.ActualValue;
       IItemList<ISingleObjectiveProblem> problems = ProblemsParameter.ActualValue;
-      ParameterConfigurationTree parameterConfiguration = ParameterConfigurationParameter.ActualValue;
+      ItemDictionary<StringValue, RunCollection> runsCache = ResultsParameter.ActualValue.ContainsKey("Runs") ? (ItemDictionary<StringValue, RunCollection>)ResultsParameter.ActualValue["Runs"].Value : null;
+      double[] referenceQualities = GetReferenceQualities(problems);
 
-      double[] referenceQualities;
-      if (ProblemQualityReferencesParameter.ActualValue == null) {
-        // this is generation zero. no reference qualities for normalization have been calculated yet.
-        referenceQualities = new double[problems.Count];
-        for (int i = 0; i < referenceQualities.Length; i++) {
-          referenceQualities[i] = 1;
-        }
+      RunCollection runs;
+      if (runsCache != null && runsCache.Count(x => x.Key.Value == parameterConfiguration.ParameterInfoString) > 0) {
+        runs = runsCache.Single(x => x.Key.Value == parameterConfiguration.ParameterInfoString).Value;
+        Console.WriteLine("Used Cache for {0}", parameterConfiguration.ParameterInfoString);
       } else {
-        referenceQualities = ProblemQualityReferencesParameter.ActualValue.ToArray();
+        runs = ExecuteAlgorithm(parameterConfiguration, algorithm, problems);
       }
-
-      // set parameters
-      parameterConfiguration.Parameterize(algorithm);
-      algorithm.StoreAlgorithmInEachRun = false;
 
       List<List<double>> qualities = new List<List<double>>();
       List<List<TimeSpan>> executionTimes = new List<List<TimeSpan>>();
-      algorithm.Engine = new SequentialEngine.SequentialEngine();
-      algorithm.Prepare(true);
 
-      foreach (ISingleObjectiveProblem problem in problems) {
-        algorithm.Problem = (IProblem)problem.Clone();
-        var problemQualities = new List<double>();
-        var problemExecutionTimes = new List<TimeSpan>();
-
-        for (int i = 0; i < Repetitions.Value; i++) {
-          algorithm.Prepare();
-
-          AlgorithmExecutor executor = new AlgorithmExecutor(algorithm);
-          executor.StartSync();
-
-          if (algorithm.ExecutionState == ExecutionState.Paused) {
-            // this parametercombination was bad. set penalty for this solution
-            problemQualities.Add(PenaltyQuality); // todo: respect Maximization; problem: this messes up average value; solution: use currently worst quality*2
-            problemExecutionTimes.Add(algorithm.ExecutionTime);
-          } else {
-            problemQualities.Add(((DoubleValue)algorithm.Results["BestQuality"].Value).Value);
-            problemExecutionTimes.Add(algorithm.ExecutionTime);
-
-            // parameters will be stored in ParameterConfigurationTree anyway. they would be redundant in runs
-            algorithm.Runs.Last().Parameters.Clear();
-            // but keep the problem, since this differs in runs
-            algorithm.Runs.Last().Parameters.Add("Problem", problem);
-          }
-        }
-        qualities.Add(problemQualities);
-        executionTimes.Add(problemExecutionTimes);
+      for (int i = 0; i < problems.Count; i++) {
+        var problemRuns = runs.Where(x => ((IntValue)x.Results["Meta.ProblemIndex"]).Value == i + 1);
+        qualities.Add(problemRuns.Select(x => ((DoubleValue)x.Results["BestQuality"]).Value).ToList());
+        executionTimes.Add(problemRuns.Select(x => ((TimeSpanValue)x.Results["Execution Time"]).Value).ToList());
       }
-      algorithm.Prepare();
 
       parameterConfiguration.AverageExecutionTimes = new ItemList<TimeSpanValue>(executionTimes.Select(t => new TimeSpanValue(TimeSpan.FromMilliseconds(t.Average(ts => ts.TotalMilliseconds)))));
@@ -129 +106 @@
       parameterConfiguration.QualityVariances = new DoubleArray(qualities.Select(q => q.Variance()).ToArray());
       parameterConfiguration.QualityStandardDeviations = new DoubleArray(qualities.Select(q => q.StandardDeviation()).ToArray());
-      parameterConfiguration.Runs = (RunCollection)algorithm.Runs.Clone();
+      parameterConfiguration.Runs = (RunCollection)runs.Clone();
 
       this.QualityParameter.ActualValue = new DoubleValue(NormalizeQualities(parameterConfiguration, referenceQualities));
 
       return base.Apply();
+    }
+
+    private double[] GetReferenceQualities(IItemList<ISingleObjectiveProblem> problems) {
+      double[] referenceQualities;
+      if (ProblemQualityReferencesParameter.ActualValue == null) {
+        // this is generation zero. no reference qualities for normalization have been calculated yet. in this special case the ReferenceQualityAnalyzer will do the normalization
+        referenceQualities = new double[problems.Count];
+        for (int i = 0; i < referenceQualities.Length; i++) {
+          referenceQualities[i] = 1;
+        }
+      } else {
+        referenceQualities = ProblemQualityReferencesParameter.ActualValue.ToArray();
+      }
+      return referenceQualities;
+    }
+
+    private RunCollection ExecuteAlgorithm(ParameterConfigurationTree parameterConfiguration, EngineAlgorithm algorithm, IItemList<ISingleObjectiveProblem> problems) {
+      EngineAlgorithm algorithmClone = (EngineAlgorithm)algorithm.Clone();
+
+      // set parameters
+      parameterConfiguration.Parameterize(algorithmClone);
+      algorithmClone.StoreAlgorithmInEachRun = false;
+
+      algorithmClone.Engine = new SequentialEngine.SequentialEngine();
+      algorithmClone.Prepare(true);
+
+      foreach (ISingleObjectiveProblem problem in problems) {
+        algorithmClone.Problem = (IProblem)problem.Clone();
+
+        for (int i = 0; i < Repetitions.Value; i++) {
+          algorithmClone.Prepare();
+
+          AlgorithmExecutor executor = new AlgorithmExecutor(algorithmClone);
+          executor.StartSync();
+
+          if (algorithmClone.ExecutionState == ExecutionState.Paused) {
+            // this parametercombination was bad. set penalty for this solution
+            // TODO!
+          }
+          int problemIndex = problems.IndexOf(problem) + 1;
+          IRun run = algorithmClone.Runs.Last();
+          run.Results.Add("Meta.FromCache", new BoolValue(false));
+          run.Results.Add("Meta.Generation", new IntValue(GenerationsParameter.ActualValue != null ? GenerationsParameter.ActualValue.Value : 0));
+          run.Results.Add("Meta.ProblemIndex", new IntValue(problemIndex));
+          int runCountOfThisProblem = algorithmClone.Runs.Where(x => ((IntValue)x.Results["Meta.ProblemIndex"]).Value == problemIndex).Count();
+          run.Name = string.Format("{0} Problem {1} Run {2}", parameterConfiguration.ParameterInfoString, problemIndex, runCountOfThisProblem);
+        }
+      }
+      algorithmClone.Prepare();
+      return algorithmClone.Runs;
     }