Changeset 6018 for branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/AlgorithmRunsAnalyzer.cs

Timestamp:

04/17/11 18:22:28 (13 years ago)

Author:

cneumuel

Message:

#1215

• support for maximization problems
• made base level algorithms stoppable
• optimization for multiple goals possible (AverageQuality, AverageDeviation, AverageEvaluatedSolutions)
• lots of fixes

File:

• branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/AlgorithmRunsAnalyzer.cs (modified) (9 diffs)

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/AlgorithmRunsAnalyzer.cs

@@ -2 +2 @@
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
 using HeuristicLab.Operators;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using HeuristicLab.Core;
-using HeuristicLab.Common;
-using HeuristicLab.Parameters;
-using HeuristicLab.Data;
-using HeuristicLab.Optimization;
 
 namespace HeuristicLab.Problems.MetaOptimization {
@@ -32 +31 @@
       get { return (ILookupParameter<IItemList<IProblem>>)Parameters[MetaOptimizationProblem.ProblemsParameterName]; }
     }
-    public LookupParameter<DoubleArray> ProblemQualityReferencesParameter {
-      get { return (LookupParameter<DoubleArray>)Parameters["ProblemQualityReferences"]; }
+    public LookupParameter<DoubleArray> ReferenceQualityAveragesParameter {
+      get { return (LookupParameter<DoubleArray>)Parameters["ReferenceQualityAverages"]; }
+    }
+    public LookupParameter<DoubleArray> ReferenceQualityDeviationsParameter {
+      get { return (LookupParameter<DoubleArray>)Parameters["ReferenceQualityDeviations"]; }
+    }
+    public LookupParameter<DoubleArray> ReferenceEvaluatedSolutionAveragesParameter {
+      get { return (LookupParameter<DoubleArray>)Parameters["ReferenceEvaluatedSolutionAverages"]; }
     }
     public LookupParameter<ResultCollection> ResultsParameter {
@@ -47 +52 @@
       get { return (ScopeTreeLookupParameter<IntValue>)Parameters["RepetitionIndex"]; }
     }
+    public LookupParameter<BoolValue> MaximizationParameter {
+      get { return (LookupParameter<BoolValue>)Parameters["Maximization"]; }
+    }
     #endregion
 
@@ -58 +66 @@
       Parameters.Add(new LookupParameter<ParameterConfigurationTree>("ParameterConfigurationTree", ""));
       Parameters.Add(new LookupParameter<IItemList<IProblem>>(MetaOptimizationProblem.ProblemsParameterName, ""));
-      Parameters.Add(new LookupParameter<DoubleArray>("ProblemQualityReferences", ""));
+      Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityAverages", ""));
+      Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityDeviations", ""));
+      Parameters.Add(new LookupParameter<DoubleArray>("ReferenceEvaluatedSolutionAverages", ""));
       Parameters.Add(new LookupParameter<ResultCollection>("Results", ""));
       Parameters.Add(new ScopeTreeLookupParameter<IAlgorithm>("Algorithm", "The finished algorithms containing Runs."));
       Parameters.Add(new ScopeTreeLookupParameter<IntValue>("ProblemIndex", "The index of the problem an algorithm was executed with."));
       Parameters.Add(new ScopeTreeLookupParameter<IntValue>("RepetitionIndex", "The index of the repetition"));
+      Parameters.Add(new LookupParameter<BoolValue>("Maximization", "Set to false if the problem should be minimized."));
     }
     protected AlgorithmRunsAnalyzer(AlgorithmRunsAnalyzer original, Cloner cloner)
@@ -78 +89 @@
       IEnumerable<string> parameterNames = parameterConfiguration.GetOptimizedParameterNames();
       IItemList<IProblem> problems = ProblemsParameter.ActualValue;
+      bool maximization = MaximizationParameter.ActualValue.Value;
       int repetitions = RepetitionsParameter.ActualValue.Value;
-      var resultNames = new List<string> { "BestQuality", "Execution Time" };
+      var resultNames = new List<string> { "BestQuality", "Execution Time", "EvaluatedSolutions" };
       int currentGeneration = GenerationsParameter.ActualValue != null ? GenerationsParameter.ActualValue.Value : 0;
-      double[] referenceQualities = GetReferenceQualities(problems.Count);
+      double[] referenceQualityAverages;
+      double[] referenceQualityDeviations;
+      double[] referenceEvaluatedSolutionAverages;
+      GetReferenceValues(problems.Count, out referenceQualityAverages, out referenceQualityDeviations, out referenceEvaluatedSolutionAverages);
+
       ResultCollection results = ResultsParameter.ActualValue;
 
@@ -88 +104 @@
         var qualities = new double[problems.Count][];
         var executionTimes = new TimeSpan[problems.Count][];
+        var evaluatedSolutions = new int[problems.Count][];
 
         for (int i = 0; i < problems.Count; i++) {
           qualities[i] = new double[repetitions];
+          evaluatedSolutions[i] = new int[repetitions];
           executionTimes[i] = new TimeSpan[repetitions];
         }
@@ -98 +116 @@
           int repetitionIndex = repetitionIndices[i].Value;
 
-          IRun run = algorithms[i].Runs.First();
+          IRun run = algorithms[i].Runs.Single();
           MetaOptimizationUtil.ClearResults(run, resultNames);
           MetaOptimizationUtil.ClearParameters(run, parameterNames);
@@ -107 +125 @@
           qualities[problemIndex][repetitionIndex] = (((DoubleValue)run.Results["BestQuality"]).Value);
           executionTimes[problemIndex][repetitionIndex] = (((TimeSpanValue)run.Results["Execution Time"]).Value);
+          evaluatedSolutions[problemIndex][repetitionIndex] = (((IntValue)run.Results["EvaluatedSolutions"]).Value);
           runs.Add((IRun)run.Clone());
         }
 
         parameterConfiguration.AverageExecutionTimes = new ItemList<TimeSpanValue>(executionTimes.Select(t => new TimeSpanValue(TimeSpan.FromMilliseconds(t.Average(ts => ts.TotalMilliseconds)))));
+        parameterConfiguration.AverageEvaluatedSolutions = new DoubleArray(evaluatedSolutions.Select(x => x.Average()).ToArray());
         parameterConfiguration.Repetitions = new IntValue(repetitions);
-        parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray()); // todo: respect Maximization:true/false
         parameterConfiguration.AverageQualities = new DoubleArray(qualities.Select(q => q.Average()).ToArray());
-        parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray()); // todo: respect Maximization:true/false
+
+        if(maximization)
+          parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray());
+        else
+          parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray());
+        
+        if (maximization)
+          parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray());
+        else
+          parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray());
+
         parameterConfiguration.QualityVariances = new DoubleArray(qualities.Select(q => q.Variance()).ToArray());
         parameterConfiguration.QualityStandardDeviations = new DoubleArray(qualities.Select(q => q.StandardDeviation()).ToArray());
         parameterConfiguration.Runs = runs;
 
-        this.QualityParameter.ActualValue = new DoubleValue(MetaOptimizationUtil.NormalizeQualities(parameterConfiguration, referenceQualities));
+        this.QualityParameter.ActualValue = new DoubleValue(MetaOptimizationUtil.Normalize(parameterConfiguration, referenceQualityAverages, referenceQualityDeviations, referenceEvaluatedSolutionAverages, 1, 1, 1, maximization));
       } else {
         // something terrible happened -> most probably due to invalid parameters.
         // penalty with worst quality from latest generation!
 
-        double penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"]).Value : referenceQualities.Max(); // todo: respect min/max
+        double penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"]).Value : referenceQualityAverages.Max(); // todo: respect min/max
         this.QualityParameter.ActualValue = new DoubleValue(penaltyValue);
       }
@@ -131 +160 @@
     }
 
-    private double[] GetReferenceQualities(int problemsCount) {
-      double[] referenceQualities;
-      if (ProblemQualityReferencesParameter.ActualValue == null) {
+    private void GetReferenceValues(int problemsCount, out double[] referenceQualityAverages, out double[] referenceQualityDeviations, out double[] referenceEvaluatedSolutionAverages) {
+      if (ReferenceQualityAveragesParameter.ActualValue == null) {
         // this is generation zero. no reference qualities for normalization have been calculated yet. in this special case the ReferenceQualityAnalyzer will do the normalization
-        referenceQualities = new double[problemsCount];
-        for (int i = 0; i < referenceQualities.Length; i++) {
-          referenceQualities[i] = 1;
+        referenceQualityAverages = new double[problemsCount];
+        referenceQualityDeviations = new double[problemsCount];
+        referenceEvaluatedSolutionAverages = new double[problemsCount];
+        for (int i = 0; i < referenceQualityAverages.Length; i++) {
+          referenceQualityAverages[i] = 1;
+          referenceQualityDeviations[i] = 1;
+          referenceEvaluatedSolutionAverages[i] = 1;
         }
       } else {
-        referenceQualities = ProblemQualityReferencesParameter.ActualValue.ToArray();
+        referenceQualityAverages = ReferenceQualityAveragesParameter.ActualValue.ToArray();
+        referenceQualityDeviations = ReferenceQualityDeviationsParameter.ActualValue.ToArray();
+        referenceEvaluatedSolutionAverages = ReferenceEvaluatedSolutionAveragesParameter.ActualValue.ToArray();
       }
-      return referenceQualities;
     }
   }