Changeset 5023 for branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/MetaOptimizationEvaluator.cs

Timestamp:

12/04/10 18:08:43 (13 years ago)

Author:

cneumuel

Message:

#1215 worked on metaoptimization

File:

• branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/MetaOptimizationEvaluator.cs (modified) (4 diffs)

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/MetaOptimizationEvaluator.cs

@@ -1 +1 @@
 using System;
+using System.Linq;
 using System.Threading;
 using HeuristicLab.Common;
@@ -8 +9 @@
 using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using System.Collections.Generic;
 
 namespace HeuristicLab.Problems.MetaOptimization {
@@ -69 +71 @@
       algorithm.Stopped += new EventHandler(ActualValue_Stopped);
 
-      double qualitySum = 0;
+      List<double> qualities = new List<double>();
+      List<TimeSpan> executionTimes = new List<TimeSpan>();
+      algorithm.Prepare(true);
 
-      //foreach (ISingleObjectiveProblem problem in ProblemsParameter.ActualValue) {
-      algorithm.Engine = new SequentialEngine.SequentialEngine();
-      //algorithm.Problem = problem;
-      algorithm.Prepare();
-      algorithm.Start();
-      while (!algorithmStopped) {
-        Thread.Sleep(1000); // wait for algorithm to complete; do not rely on Algorithm.ExecutionState here, because change of ExecutionState happens before Run is added (which causes problems because Algorithm might get cloned when its started already)
+      for (int i = 0; i < Repetitions.Value; i++) {
+        algorithm.Engine = new SequentialEngine.SequentialEngine();
+        algorithm.Prepare();
+        algorithm.Start();
+        while (!algorithmStopped) {
+          Thread.Sleep(200); // wait for algorithm to complete; do not rely on Algorithm.ExecutionState here, because change of ExecutionState happens before Run is added (which causes problems because Algorithm might get cloned when its started already)
+        }
+        qualities.Add(((DoubleValue)algorithm.Results["BestQuality"].Value).Value);
+        executionTimes.Add(algorithm.ExecutionTime);
+
+        algorithmStopped = false;
       }
-      double quality = ((DoubleValue)algorithm.Results["BestQuality"].Value).Value;
-      ParameterConfigurationParameter.ActualValue.Quality.Value = quality;
-      qualitySum += quality;
-      //}
-
       algorithm.Stopped -= new EventHandler(ActualValue_Stopped);
 
-      //double qualityAvg = qualitySum / ProblemsParameter.ActualValue.Count;
-      //this.QualityParameter.ActualValue = new DoubleValue(qualityAvg);
+      qualities = qualities.OrderBy(x => x).ToList();  // todo: respect Maximization:true/false
+      
+      ParameterConfigurationParameter.ActualValue.AverageExecutionTime = new TimeSpanValue(TimeSpan.FromMilliseconds(executionTimes.Average(t => t.TotalMilliseconds)));
+      ParameterConfigurationParameter.ActualValue.Repetitions = Repetitions;
+      ParameterConfigurationParameter.ActualValue.BestQuality = new DoubleValue(qualities.First());
+      ParameterConfigurationParameter.ActualValue.AverageQuality = new DoubleValue(qualities.Average());
+      ParameterConfigurationParameter.ActualValue.WorstQuality = new DoubleValue(qualities.Last());
+      ParameterConfigurationParameter.ActualValue.QualityVariance = new DoubleValue(qualities.Variance());
+      ParameterConfigurationParameter.ActualValue.QualityStandardDeviation = new DoubleValue(qualities.StandardDeviation());
+      ParameterConfigurationParameter.ActualValue.Runs = algorithm.Runs;
+
+      double quality = ParameterConfigurationParameter.ActualValue.AverageQuality.Value; // todo: also include other measures (executiontime, variance)
       this.QualityParameter.ActualValue = new DoubleValue(quality);
+      
+      return base.Apply();
+    }
 
-      return base.Apply();
+    public static double Variance(IEnumerable<double> source) {
+      double avg = source.Average();
+      double d = source.Aggregate(0.0, (total, next) => total += Math.Pow(next - avg, 2));
+      return d / (source.Count() - 1);
+    }
+
+    public static double StandardDeviation(IEnumerable<double> source) {
+      return Math.Sqrt(source.Variance());
     }
 
@@ -96 +119 @@
       algorithmStopped = true;
     }
-
-    //private void ParameterizeAlgorithm() {
-    //  foreach (IParameterConfiguration parameter in ParameterVectorParameter.ActualValue) {
-    //    if (typeof(IAlgorithm).IsAssignableFrom(parameter.OperatorType)) {
-    //      this.AlgorithmParameter.ActualValue.Parameters[parameter.Parameter.Name].ActualValue = parameter.Parameter.ActualValue;
-    //    } else if (typeof(IProblem).IsAssignableFrom(parameter.OperatorType)) {
-    //      this.AlgorithmParameter.ActualValue.Problem.Parameters[parameter.Parameter.Name].ActualValue = parameter.Parameter.ActualValue;
-    //    }
-    //  }
-    //}
   }
 }