Changeset 6018 for branches/HeuristicLab.MetaOptimization

Timestamp:

04/17/11 18:22:28 (14 years ago)

Author:

cneumuel

Message:

#1215

• support for maximization problems
• made base level algorithms stoppable
• optimization for multiple goals possible (AverageQuality, AverageDeviation, AverageEvaluatedSolutions)
• lots of fixes

Location:

branches/HeuristicLab.MetaOptimization

Files:

• HeuristicLab.MetaOptimization.Test/Program.cs (modified) (5 diffs)
• HeuristicLab.Problems.MetaOptimization/3.3/AlgorithmExecutor.cs (modified) (3 diffs)
• HeuristicLab.Problems.MetaOptimization/3.3/Analyzers/BestParameterConfigurationAnalyzer.cs (modified) (6 diffs)
• HeuristicLab.Problems.MetaOptimization/3.3/Analyzers/PMOBestSolutionHistoryAnalyzer.cs (modified) (4 diffs)
• HeuristicLab.Problems.MetaOptimization/3.3/Analyzers/PMOProblemQualitiesAnalyzer.cs (modified) (5 diffs)
• HeuristicLab.Problems.MetaOptimization/3.3/Analyzers/ReferenceQualityAnalyzer.cs (modified) (6 diffs)
• HeuristicLab.Problems.MetaOptimization/3.3/Analyzers/SolutionCacheAnalyzer.cs (modified) (3 diffs)
• HeuristicLab.Problems.MetaOptimization/3.3/Encoding/ParameterConfigurationTree.cs (modified) (12 diffs)
• HeuristicLab.Problems.MetaOptimization/3.3/Encoding/ParameterConfigurations/ParameterConfiguration.cs (modified) (7 diffs)
• HeuristicLab.Problems.MetaOptimization/3.3/Encoding/ParameterConfigurations/SingleValuedParameterConfiguration.cs (modified) (2 diffs)
• HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/AlgorithmEvaluator.cs (modified) (1 diff)
• HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/AlgorithmRunsAnalyzer.cs (modified) (9 diffs)
• HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/AlgorithmSubScopesCreator.cs (modified) (1 diff)
• HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/PMOEvaluator.cs (modified) (3 diffs)
• HeuristicLab.Problems.MetaOptimization/3.3/MetaOptimizationUtil.cs (modified) (2 diffs)

branches/HeuristicLab.MetaOptimization/HeuristicLab.MetaOptimization.Test/Program.cs

@@ -47 +47 @@
     //private static int baseAlgorithmMaxGenerations = 1000;
 
-    private static int metaAlgorithmPopulationSize = 20;
-    private static int metaAlgorithmMaxGenerations = 20;
-    private static int metaProblemRepetitions = 3;
-    private static int baseAlgorithmMaxGenerations = 100;
+    private static int metaAlgorithmPopulationSize = 10;
+    private static int metaAlgorithmMaxGenerations = 10;
+    private static int metaProblemRepetitions = 2;
+    private static int baseAlgorithmMaxGenerations = 10;
     private static double mutationProbability = 0.10;
 
@@ -80 +80 @@
       MetaOptimizationProblem metaOptimizationProblem = new MetaOptimizationProblem();
       metaOptimizationProblem.Repetitions = new IntValue(metaProblemRepetitions);
-      //GeneticAlgorithm metaLevelAlgorithm = GetMetaGA(metaOptimizationProblem);
-      GeneticAlgorithm metaLevelAlgorithm = GetParallelMetaGA(metaOptimizationProblem);
+      GeneticAlgorithm metaLevelAlgorithm = GetMetaGA(metaOptimizationProblem);
+      //GeneticAlgorithm metaLevelAlgorithm = GetParallelMetaGA(metaOptimizationProblem);
       //GeneticAlgorithm metaLevelAlgorithm = GetHiveParallelMetaGA(metaOptimizationProblem);
 
@@ -88 +88 @@
       var algorithmVc = SetupGAAlgorithm(typeof(GeneticAlgorithm), metaOptimizationProblem);
 
-      //TestToString(algorithmVc);
-
+      string info = algorithmVc.ParameterInfoString;
 
       //Console.WriteLine("Press enter to start");
@@ -116 +115 @@
 
       var vc1 = new SymbolicExpressionGrammarValueConfiguration(grammar1);
+
+      string info = vc1.ParameterInfoString;
+
       ConfigureSymbolicExpressionGrammarVc(vc1);
+
+      info = vc1.ParameterInfoString;
+
+
       var vc2 = new SymbolicExpressionGrammarValueConfiguration(grammar2);
       ConfigureSymbolicExpressionGrammarVc(vc2);
@@ -758 +764 @@
           && pc.Name != "StartSymbol") {
           pc.Optimize = true;
-          var initialFrequencyVc = ((ParameterizedValueConfiguration)pc.ValueConfigurations.First()).ParameterConfigurations.Single(x => x.Name == "InitialFrequency");
-          initialFrequencyVc.Optimize = true;
         }
       }

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/AlgorithmExecutor.cs

@@ -10 +10 @@
       executor.StartSync();
     }
+    public static void StartSync(this IAlgorithm algorithm, CancellationToken cancellationToken) {
+      var executor = new AlgorithmExecutor(algorithm, cancellationToken);
+      executor.StartSync();
+    }
   }
 
@@ -18 +22 @@
     private IAlgorithm algorithm;
     private AutoResetEvent waitHandle = new AutoResetEvent(false);
+    private CancellationToken cancellationToken;
+    private bool useCancellationToken = false;
 
     public AlgorithmExecutor(IAlgorithm algorithm) {
@@ -23 +29 @@
     }
 
+    public AlgorithmExecutor(IAlgorithm algorithm, CancellationToken cancellationToken) {
+      this.algorithm = algorithm;
+      this.cancellationToken = cancellationToken;
+      this.useCancellationToken = true;
+    }
+
     public void StartSync() {
       algorithm.Stopped += new EventHandler(algorithm_Stopped);
       algorithm.Paused += new EventHandler(algorithm_Paused);
-      algorithm.Start();
+      
+      if(useCancellationToken && algorithm is EngineAlgorithm && ((EngineAlgorithm)algorithm).Engine is SequentialEngine.SequentialEngine) {
+        ((SequentialEngine.SequentialEngine)((EngineAlgorithm)algorithm).Engine).Start(cancellationToken);
+      } else {
+        algorithm.Start();
+      }
+
       waitHandle.WaitOne();
       waitHandle.Dispose();

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Analyzers/BestParameterConfigurationAnalyzer.cs

@@ -7 +7 @@
 using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using System.Collections.Generic;
-using System;
 
 namespace HeuristicLab.Problems.MetaOptimization {
@@ -39 +37 @@
       get { return (LookupParameter<RunCollection>)Parameters["Population"]; }
     }
-    public LookupParameter<DoubleArray> ProblemQualityReferencesParameter {
-      get { return (LookupParameter<DoubleArray>)Parameters["ProblemQualityReferences"]; }
-    }
     public LookupParameter<ConstrainedItemList<IProblem>> ProblemsParameter {
       get { return (LookupParameter<ConstrainedItemList<IProblem>>)Parameters[MetaOptimizationProblem.ProblemsParameterName]; }
+    }
+    public LookupParameter<BoolValue> MaximizationParameter {
+      get { return (LookupParameter<BoolValue>)Parameters["Maximization"]; }
     }
 
@@ -55 +53 @@
       Parameters.Add(new LookupParameter<IRun>("BestKnownSolution", ""));
       Parameters.Add(new LookupParameter<RunCollection>("Population", ""));
-      Parameters.Add(new LookupParameter<DoubleArray>("ProblemQualityReferences", ""));
       Parameters.Add(new LookupParameter<ConstrainedItemList<IProblem>>(MetaOptimizationProblem.ProblemsParameterName));
+      Parameters.Add(new LookupParameter<BoolValue>("Maximization", "Set to false if the problem should be minimized."));
     }
 
@@ -71 +69 @@
       DoubleValue bestKnownQuality = BestKnownQualityParameter.ActualValue;
       ItemArray<ParameterConfigurationTree> parameterTrees = ParameterConfigurationParameter.ActualValue;
+      bool maximization = MaximizationParameter.ActualValue.Value;
 
-      int idxBest = qualities.Select((x, index) => new { index, x.Value }).OrderBy(x => x.Value).First().index;
+      int idxBest;
+      if (maximization)
+        idxBest = qualities.Select((x, index) => new { index, x.Value }).OrderBy(x => x.Value).Last().index;
+      else
+        idxBest = qualities.Select((x, index) => new { index, x.Value }).OrderBy(x => x.Value).First().index;
 
       ParameterConfigurationTree best = (ParameterConfigurationTree)parameterTrees[idxBest];
@@ -79 +82 @@
       best.CollectParameterValues(bestRun.Parameters);
 
-      if (bestKnownQuality == null || qualities[idxBest].Value < bestKnownQuality.Value) { // todo: respect Maximization:true/false
+      if (bestKnownQuality == null || 
+        (!maximization && (qualities[idxBest].Value < bestKnownQuality.Value) ||
+        (maximization && (qualities[idxBest].Value > bestKnownQuality.Value)))) {
         BestKnownQualityParameter.ActualValue = new DoubleValue(qualities[idxBest].Value);
 
@@ -96 +101 @@
       int i = 0;
       RunCollection rc = new RunCollection();
-      foreach (ParameterConfigurationTree pt in parameterTrees.OrderBy(x => x.AverageQualityNormalized)) { // todo: respect Maximization:true/false
+      foreach (ParameterConfigurationTree pt in (maximization ? parameterTrees.OrderByDescending(x => x.Quality) : parameterTrees.OrderBy(x => x.Quality))) {
         rc.Add(pt.ToRun(string.Format("Individual {0} ({1})", i, pt.ParameterInfoString)));
         i++;

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Analyzers/PMOBestSolutionHistoryAnalyzer.cs

@@ -7 +7 @@
 using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using System.Collections.Generic;
-using System;
-using HeuristicLab.Analysis;
 
 namespace HeuristicLab.Problems.MetaOptimization {
@@ -34 +31 @@
       get { return (LookupParameter<IntValue>)Parameters["Generations"]; }
     }
+    public LookupParameter<BoolValue> MaximizationParameter {
+      get { return (LookupParameter<BoolValue>)Parameters["Maximization"]; }
+    }
 
     public PMOBestSolutionHistoryAnalyzer()
@@ -42 +42 @@
       Parameters.Add(new LookupParameter<ConstrainedItemList<IProblem>>(MetaOptimizationProblem.ProblemsParameterName));
       Parameters.Add(new LookupParameter<IntValue>("Generations", ""));
+      Parameters.Add(new LookupParameter<BoolValue>("Maximization", "Set to false if the problem should be minimized."));
     }
 
@@ -56 +57 @@
       ItemArray<ParameterConfigurationTree> parameterTrees = ParameterConfigurationParameter.ActualValue;
       int currentGeneration = GenerationsParameter.ActualValue.Value;
-      int idxBest = qualities.Select((x, index) => new { index, x.Value }).OrderBy(x => x.Value).First().index; // todo: respect Min/Max
+      bool maximization = MaximizationParameter.ActualValue.Value;
+
+      int idxBest;
+      if (maximization)
+        idxBest = qualities.Select((x, index) => new { index, x.Value }).OrderBy(x => x.Value).Last().index;
+      else
+        idxBest = qualities.Select((x, index) => new { index, x.Value }).OrderBy(x => x.Value).First().index;
+
       ParameterConfigurationTree best = parameterTrees[idxBest];
 

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Analyzers/PMOProblemQualitiesAnalyzer.cs

@@ -1 +1 @@
 using System.Linq;
+using HeuristicLab.Analysis;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -7 +8 @@
 using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using System.Collections.Generic;
-using System;
-using HeuristicLab.Analysis;
 
 namespace HeuristicLab.Problems.MetaOptimization {
@@ -31 +29 @@
       get { return (LookupParameter<ConstrainedItemList<IProblem>>)Parameters[MetaOptimizationProblem.ProblemsParameterName]; }
     }
+    public LookupParameter<BoolValue> MaximizationParameter {
+      get { return (LookupParameter<BoolValue>)Parameters["Maximization"]; }
+    }
 
     public PMOProblemQualitiesAnalyzer()
@@ -38 +39 @@
       Parameters.Add(new ValueLookupParameter<ResultCollection>("Results", ""));
       Parameters.Add(new LookupParameter<ConstrainedItemList<IProblem>>(MetaOptimizationProblem.ProblemsParameterName));
+      Parameters.Add(new LookupParameter<BoolValue>("Maximization", "Set to false if the problem should be minimized."));
     }
 
@@ -51 +53 @@
       ResultCollection results = ResultsParameter.ActualValue;
       ItemArray<ParameterConfigurationTree> parameterTrees = ParameterConfigurationParameter.ActualValue;
-      int idxBest = qualities.Select((x, index) => new { index, x.Value }).OrderBy(x => x.Value).First().index; // todo: respect Min/Max
-      int idxWorst = qualities.Select((x, index) => new { index, x.Value }).OrderBy(x => x.Value).Last().index;
-      
+      bool maximization = MaximizationParameter.ActualValue.Value;
+
+      int idxBest ;
+      int idxWorst;
+      if (maximization) {
+        idxBest = qualities.Select((x, index) => new { index, x.Value }).OrderBy(x => x.Value).Last().index;
+        idxWorst = qualities.Select((x, index) => new { index, x.Value }).OrderBy(x => x.Value).First().index;
+      } else {
+        idxBest = qualities.Select((x, index) => new { index, x.Value }).OrderBy(x => x.Value).First().index;
+        idxWorst = qualities.Select((x, index) => new { index, x.Value }).OrderBy(x => x.Value).Last().index;
+      }
+
       int problemCount = ProblemsParameter.ActualValue.Count;
 

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Analyzers/ReferenceQualityAnalyzer.cs

@@ -7 +7 @@
 using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using System.Collections.Generic;
-using System;
 
 namespace HeuristicLab.Problems.MetaOptimization {
@@ -17 +15 @@
   [StorableClass]
   public sealed class ReferenceQualityAnalyzer : SingleSuccessorOperator, IAnalyzer {
-    
+
     public ValueLookupParameter<ResultCollection> ResultsParameter {
       get { return (ValueLookupParameter<ResultCollection>)Parameters["Results"]; }
@@ -27 +25 @@
       get { return (ScopeTreeLookupParameter<DoubleValue>)Parameters["Quality"]; }
     }
-    public LookupParameter<DoubleArray> ProblemQualityReferenceParameter {
-      get { return (LookupParameter<DoubleArray>)Parameters["ProblemQualityReferences"]; }
+    public LookupParameter<DoubleArray> ReferenceQualityAveragesParameter {
+      get { return (LookupParameter<DoubleArray>)Parameters["ReferenceQualityAverages"]; }
+    }
+    public LookupParameter<DoubleArray> ReferenceQualityDeviationsParameter {
+      get { return (LookupParameter<DoubleArray>)Parameters["ReferenceQualityDeviations"]; }
+    }
+    public LookupParameter<DoubleArray> ReferenceEvaluatedSolutionAveragesParameter {
+      get { return (LookupParameter<DoubleArray>)Parameters["ReferenceEvaluatedSolutionAverages"]; }
     }
     public LookupParameter<ConstrainedItemList<IProblem>> ProblemsParameter {
       get { return (LookupParameter<ConstrainedItemList<IProblem>>)Parameters[MetaOptimizationProblem.ProblemsParameterName]; }
+    }
+    public LookupParameter<BoolValue> MaximizationParameter {
+      get { return (LookupParameter<BoolValue>)Parameters["Maximization"]; }
     }
 
@@ -39 +46 @@
       Parameters.Add(new ValueLookupParameter<ResultCollection>("Results", ""));
       Parameters.Add(new ScopeTreeLookupParameter<ParameterConfigurationTree>("ParameterConfigurationTree", ""));
-      Parameters.Add(new LookupParameter<DoubleArray>("ProblemQualityReferences", ""));
+      Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityAverages", ""));
+      Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityDeviations", ""));
+      Parameters.Add(new LookupParameter<DoubleArray>("ReferenceEvaluatedSolutionAverages", ""));
       Parameters.Add(new LookupParameter<ConstrainedItemList<IProblem>>(MetaOptimizationProblem.ProblemsParameterName, ""));
+      Parameters.Add(new LookupParameter<BoolValue>("Maximization", "Set to false if the problem should be minimized."));
     }
 
@@ -54 +64 @@
       ItemArray<ParameterConfigurationTree> solutions = ParameterConfigurationParameter.ActualValue;
       ItemArray<DoubleValue> qualities = QualityParameter.ActualValue;
+      bool maximization = MaximizationParameter.ActualValue.Value;
 
-      if (ProblemQualityReferenceParameter.ActualValue == null) {
-        DoubleArray referenceQualities = CalculateReferenceQualities(solutions);
-        ProblemQualityReferenceParameter.ActualValue = referenceQualities;
-        NormalizePopulation(solutions, qualities, referenceQualities);
-        results.Add(new Result("QualityReferences", referenceQualities));
+      if (ReferenceQualityAveragesParameter.ActualValue == null) {
+        // this is generation zero. calculate the reference values and apply them on population. in future generations `AlgorithmRunsAnalyzer` will do the nomalization
+        DoubleArray referenceQualityAverages = CalculateReferenceQualityAverages(solutions, maximization);
+        DoubleArray referenceQualityDeviations = CalculateReferenceQualityDeviations(solutions, maximization);
+        DoubleArray referenceEvaluatedSolutionAverages = CalculateReferenceEvaluatedSolutionAverages(solutions, maximization);
+
+        ReferenceQualityAveragesParameter.ActualValue = referenceQualityAverages;
+        ReferenceQualityDeviationsParameter.ActualValue = referenceQualityDeviations;
+        ReferenceEvaluatedSolutionAveragesParameter.ActualValue = referenceEvaluatedSolutionAverages;
+
+        NormalizePopulation(solutions, qualities, referenceQualityAverages, referenceQualityDeviations, referenceEvaluatedSolutionAverages, 1, 1, 1, maximization);
+
+        results.Add(new Result("ReferenceQualities", referenceQualityAverages));
+        results.Add(new Result("ReferenceQualityDeviations", referenceQualityDeviations));
+        results.Add(new Result("ReferenceEvaluatedSolutionAverages", referenceEvaluatedSolutionAverages));
       }
 
@@ -65 +86 @@
     }
 
-    private DoubleArray CalculateReferenceQualities(ItemArray<ParameterConfigurationTree> solutions) {
-      DoubleArray referenceQualities = new DoubleArray(ProblemsParameter.ActualValue.Count);
+    private DoubleArray CalculateReferenceQualityAverages(ItemArray<ParameterConfigurationTree> solutions, bool maximization) {
+      DoubleArray references = new DoubleArray(ProblemsParameter.ActualValue.Count);
       for (int pi = 0; pi < ProblemsParameter.ActualValue.Count; pi++) {
-        referenceQualities[pi] = solutions.Where(x => x.AverageQualities != null).Select(x => x.AverageQualities[pi]).Min(); // todo: respect minimization / maximization
+        if (maximization)
+          references[pi] = solutions.Where(x => x.AverageQualities != null).Select(x => x.AverageQualities[pi]).Max();
+        else
+          references[pi] = solutions.Where(x => x.AverageQualities != null).Select(x => x.AverageQualities[pi]).Min();
       }
-      return referenceQualities;
+      return references;
     }
 
-    private void NormalizePopulation(ItemArray<ParameterConfigurationTree> solutions, ItemArray<DoubleValue> qualities, DoubleArray referenceQualities) {
+    private DoubleArray CalculateReferenceQualityDeviations(ItemArray<ParameterConfigurationTree> solutions, bool maximization) {
+      DoubleArray references = new DoubleArray(ProblemsParameter.ActualValue.Count);
+      for (int pi = 0; pi < ProblemsParameter.ActualValue.Count; pi++) {
+        if (maximization)
+          references[pi] = solutions.Where(x => x.QualityStandardDeviations != null).Select(x => x.QualityStandardDeviations[pi]).Max();
+        else
+          references[pi] = solutions.Where(x => x.QualityStandardDeviations != null).Select(x => x.QualityStandardDeviations[pi]).Min();
+      }
+      return references;
+    }
+
+    private DoubleArray CalculateReferenceEvaluatedSolutionAverages(ItemArray<ParameterConfigurationTree> solutions, bool maximization) {
+      DoubleArray references = new DoubleArray(ProblemsParameter.ActualValue.Count);
+      for (int pi = 0; pi < ProblemsParameter.ActualValue.Count; pi++) {
+        if (maximization)
+          references[pi] = solutions.Where(x => x.AverageEvaluatedSolutions != null).Select(x => x.AverageEvaluatedSolutions[pi]).Max();
+        else
+          references[pi] = solutions.Where(x => x.AverageEvaluatedSolutions != null).Select(x => x.AverageEvaluatedSolutions[pi]).Min();
+      }
+      return references;
+    }
+
+    private void NormalizePopulation(ItemArray<ParameterConfigurationTree> solutions, ItemArray<DoubleValue> qualities,
+          DoubleArray referenceQualityAverages,
+          DoubleArray referenceQualityDeviations,
+          DoubleArray referenceEvaluatedSolutionAverages,
+          double qualityAveragesWeight,
+          double qualityDeviationsWeight,
+          double evaluatedSolutionsWeight,
+          bool maximization) {
       for (int i = 0; i < solutions.Length; i++) {
-        qualities[i].Value = MetaOptimizationUtil.NormalizeQualities(solutions[i], referenceQualities.ToArray());
+        if (solutions[i].AverageQualities == null || solutions[i].QualityStandardDeviations == null || solutions[i].AverageEvaluatedSolutions == null) {
+          // this parameterConfigurationTree has not been evaluated correctly (due to a faulty configuration, which led to an exception)
+          // since we are in generation zero, there is no WorstQuality available for a penalty value
+          double penaltyValue = maximization ? double.MinValue : double.MaxValue;
+          qualities[i].Value = penaltyValue;
+        } else {
+          qualities[i].Value = MetaOptimizationUtil.Normalize(solutions[i],
+            referenceQualityAverages.ToArray(),
+            referenceQualityDeviations.ToArray(),
+            referenceEvaluatedSolutionAverages.ToArray(),
+            qualityAveragesWeight,
+            qualityDeviationsWeight,
+            evaluatedSolutionsWeight,
+            maximization);
+        }
       }
     }

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Analyzers/SolutionCacheAnalyzer.cs

@@ -7 +7 @@
 using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using System.Collections.Generic;
-using System;
 
 namespace HeuristicLab.Problems.MetaOptimization {
@@ -27 +25 @@
       get { return (ScopeTreeLookupParameter<DoubleValue>)Parameters["Quality"]; }
     }
-    public LookupParameter<DoubleArray> ProblemQualityReferenceParameter {
-      get { return (LookupParameter<DoubleArray>)Parameters["ProblemQualityReferences"]; }
-    }
     public LookupParameter<ConstrainedItemList<IProblem>> ProblemsParameter {
       get { return (LookupParameter<ConstrainedItemList<IProblem>>)Parameters[MetaOptimizationProblem.ProblemsParameterName]; }
@@ -45 +40 @@
       Parameters.Add(new ValueLookupParameter<ResultCollection>("Results", ""));
       Parameters.Add(new ScopeTreeLookupParameter<ParameterConfigurationTree>("ParameterConfigurationTree", ""));
-      Parameters.Add(new LookupParameter<DoubleArray>("ProblemQualityReferences", ""));
       Parameters.Add(new LookupParameter<ConstrainedItemList<IProblem>>(MetaOptimizationProblem.ProblemsParameterName, ""));
       Parameters.Add(new LookupParameter<IntValue>("Generations", ""));

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Encoding/ParameterConfigurationTree.cs

@@ -16 +16 @@
 
     [Storable]
-    private DoubleValue averageQualityNormalized;
-    public DoubleValue AverageQualityNormalized {
-      get { return averageQualityNormalized; }
-      set {
-        if (averageQualityNormalized != value) {
-          averageQualityNormalized = value;
+    private DoubleValue quality;
+    public DoubleValue Quality {
+      get { return quality; }
+      set {
+        if (quality != value) {
+          quality = value;
           OnQualityChanged();
         }
@@ -28 +28 @@
 
     [Storable]
-    private DoubleArray qualitiesNormalized;
-    public DoubleArray QualitiesNormalized {
-      get { return qualitiesNormalized; }
-      set {
-        if (qualitiesNormalized != value) {
-          qualitiesNormalized = value;
-        }
-      }
-    }
-
+    private DoubleArray normalizedQualityAverages;
+    public DoubleArray NormalizedQualityAverages {
+      get { return normalizedQualityAverages; }
+      set {
+        if (normalizedQualityAverages != value) {
+          normalizedQualityAverages = value;
+        }
+      }
+    }
+
+    [Storable]
+    private DoubleArray normalizedQualityDeviations;
+    public DoubleArray NormalizedQualityDeviations {
+      get { return normalizedQualityDeviations; }
+      set {
+        if (normalizedQualityDeviations != value) {
+          normalizedQualityDeviations = value;
+        }
+      }
+    }
+
+    [Storable]
+    private DoubleArray normalizedEvaluatedSolutions;
+    public DoubleArray NormalizedEvaluatedSolutions {
+      get { return normalizedEvaluatedSolutions; }
+      set {
+        if (normalizedEvaluatedSolutions != value) {
+          normalizedEvaluatedSolutions = value;
+        }
+      }
+    }
+    
     [Storable]
     private DoubleArray bestQualities;
@@ -85 +107 @@
 
     [Storable]
+    private DoubleArray averageEvaluatedSolutions;
+    public DoubleArray AverageEvaluatedSolutions {
+      get { return averageEvaluatedSolutions; }
+      set { averageEvaluatedSolutions = value; }
+    }
+
+    [Storable]
     private IntValue repetitions;
     public IntValue Repetitions {
@@ -116 +145 @@
       }
     }
-
-    //[Storable]
-    //protected string name;
-    //public string Name {
-    //  get { return name; }
-    //  set { name = value; }
-    //}
 
     #region constructors and cloning
@@ -136 +158 @@
       algproblemitem.ProblemParameter.Value = problem;
       this.discoverValidValues = false;
-      
+
       this.parameterConfigurations.Add(new SingleValuedParameterConfiguration("Algorithm", algproblemitem.AlgorithmParameter));
       this.parameterConfigurations.Add(new SingleValuedParameterConfiguration("Problem", algproblemitem.ProblemParameter));
@@ -145 +167 @@
     protected ParameterConfigurationTree(ParameterConfigurationTree original, Cloner cloner)
       : base(original, cloner) {
-      this.averageQualityNormalized = cloner.Clone(original.averageQualityNormalized);
-      this.qualitiesNormalized = cloner.Clone(original.qualitiesNormalized);
+      this.quality = cloner.Clone(original.quality);
+      this.normalizedQualityAverages = cloner.Clone(original.normalizedQualityAverages);
+      this.normalizedQualityDeviations = cloner.Clone(original.normalizedQualityDeviations);
+      this.normalizedEvaluatedSolutions = cloner.Clone(original.normalizedEvaluatedSolutions);
       this.bestQualities = cloner.Clone(original.BestQualities);
       this.averageQualities = cloner.Clone(original.averageQualities);
@@ -153 +177 @@
       this.qualityVariances = cloner.Clone(original.qualityVariances);
       this.averageExecutionTimes = cloner.Clone(original.averageExecutionTimes);
+      this.averageEvaluatedSolutions = cloner.Clone(original.averageEvaluatedSolutions);
       this.repetitions = cloner.Clone(original.repetitions);
       this.runs = cloner.Clone(original.runs);
@@ -171 +196 @@
     public virtual void CollectResultValues(IDictionary<string, IItem> values) {
       values.Add("RunsAverageExecutionTimes", AverageExecutionTimes);
+      values.Add("RunsAverageEvaluatedSolutions", AverageEvaluatedSolutions);
       values.Add("Repetitions", Repetitions);
       values.Add("RunsBestQualities", BestQualities);
@@ -177 +203 @@
       values.Add("RunsQualityVariances", QualityVariances);
       values.Add("RunsQualityStandardDeviations", QualityStandardDeviations);
-      values.Add("QualitiesNormalized", QualitiesNormalized);
-      values.Add("AverageQualityNormalized", AverageQualityNormalized);
+      values.Add("QualitiesNormalized", NormalizedQualityAverages);
+      values.Add("AverageQualityNormalized", Quality);
       values.Add("Runs", Runs);
     }
@@ -209 +235 @@
 
       algorithm.CollectParameterValues(this.Parameters);
-      //((IProblem)ProblemConfiguration.ActualValue.Value).CollectParameterValues(this.Parameters);
-      //CollectAlgorithmParameterValues((IAlgorithm)AlgorithmConfiguration.ActualValue.Value, parameters);
-    }
-
-    //protected virtual void CollectAlgorithmParameterValues(IAlgorithm algorithm, IDictionary<string, IItem> values) {
-    //  var algorithmValues = new Dictionary<string, IItem>();
-    //  algorithm.CollectParameterValues(algorithmValues);
-    //  foreach (var value in algorithmValues) {
-    //    values.Add("Algorithm." + value.Key, value.Value);
-    //  }
-    //}
+    }
 
     public Experiment GenerateExperiment(IAlgorithm algorithm, bool createBatchRuns, int repetitions) {
@@ -294 +310 @@
         var sb = new StringBuilder();
         if (!string.IsNullOrEmpty(algorithmInfo)) {
-          sb.Append("Algorithm: ");
+          sb.Append("Algorithm (");
           sb.Append(algorithmInfo);
+          sb.Append(")");
         }
         if (!string.IsNullOrEmpty(problemInfo)) {
           if (sb.Length > 0)
             sb.Append(", ");
-          sb.Append("Problem: ");
+          sb.Append("Problem( ");
           sb.Append(problemInfo);
+          sb.Append(")");
         }
         return sb.ToString();
@@ -311 +329 @@
       ProblemConfiguration.CollectOptimizedParameterNames(parameterNames, string.Empty);
     }
-
-   
   }
 }

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Encoding/ParameterConfigurations/ParameterConfiguration.cs

@@ -92 +92 @@
           if (this.valueConfigurations != null) DeregisterValueConfigurationEvents();
           this.valueConfigurations = value;
+          KeepActualValueConfigurationIndexConsistent();
           if (this.valueConfigurations != null) RegisterValueConfigurationEvents();
         }
@@ -144 +145 @@
       set { autoPopulateValueConfigurations = value; }
     }
-    
+
 
     #region Constructors and Cloning
@@ -258 +259 @@
         } else {
           IItem val;
-          if (ActualValue.Value != null && ActualValue.ValueDataType == t) {
+          if (ActualValue.Value != null && ActualValue.Value.GetType() == t) {
             val = (IItem)ActualValue.Value.Clone(); // use existing value for that type (if available)
           } else {
@@ -363 +364 @@
     void ValueConfigurations_CheckedItemsChanged(object sender, Collections.CollectionItemsChangedEventArgs<IndexedItem<IValueConfiguration>> e) {
       OnToStringChanged();
-    }
+      KeepActualValueConfigurationIndexConsistent();
+    }
+
     void ValueConfigurations_ItemsRemoved(object sender, Collections.CollectionItemsChangedEventArgs<IndexedItem<IValueConfiguration>> e) {
       OnToStringChanged();
+      KeepActualValueConfigurationIndexConsistent();
     }
     void ValueConfigurations_ItemsAdded(object sender, Collections.CollectionItemsChangedEventArgs<IndexedItem<IValueConfiguration>> e) {
       OnToStringChanged();
+      KeepActualValueConfigurationIndexConsistent();
     }
 
@@ -440 +445 @@
     }
 
-    public string ParameterInfoString {
+    public virtual string ParameterInfoString {
       get {
         StringBuilder sb = new StringBuilder();
@@ -455 +460 @@
           }
 
-          if (this.ActualValue.Value is IParameterizedItem) {
+          if (this.ValueConfigurations[actualValueConfigurationIndex] is ParameterizedValueConfiguration) {
             string subParams = this.ValueConfigurations[actualValueConfigurationIndex].ParameterInfoString;
             if (!string.IsNullOrEmpty(subParams)) {
@@ -583 +588 @@
       }
     }
+
+    /// <summary>
+    /// Ensures that the ActualValueConfigurationIndex has a valid value
+    /// Only indices of items which are checked are allowed
+    /// </summary>
+    private void KeepActualValueConfigurationIndexConsistent() {
+      if (this.valueConfigurations != null && this.valueConfigurations.CheckedItems.Count() > 0) {
+        if(this.valueConfigurations.Count <= this.actualValueConfigurationIndex &&
+           this.valueConfigurations.CheckedItems.Count(x => x.Index == this.actualValueConfigurationIndex) == 1) {
+          // everything is ok; do nothing
+        } else {
+          // current index is invalid, set new one
+          this.ActualValueConfigurationIndex = this.valueConfigurations.CheckedItems.First().Index;
+        }
+      } else {
+        // no checked valueConfiguration is available; cannot be used
+        this.ActualValueConfigurationIndex = -1;
+      }
+    }
   }
 }

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Encoding/ParameterConfigurations/SingleValuedParameterConfiguration.cs

@@ -24 +24 @@
       this.discoverValidValues = false;
       this.validValues = new ItemSet<IItem> { valueConfiguration.ActualValue.Value };
-      this.validTypes = new Type[] {valueConfiguration.ActualValue.ValueDataType};
+      this.validTypes = new Type[] { valueConfiguration.ActualValue.ValueDataType };
       this.IsNullable = false;
       this.itemImage = valueConfiguration.ItemImage;
@@ -69 +69 @@
       OnToStringChanged();
     }
+
+    public override string ParameterInfoString {
+      get {
+        if (this.Optimize) {
+          return string.Format("{0} ({1})", this.Name, this.ValueConfigurations.Single().ParameterInfoString);
+        } else {
+          return string.Empty;
+        }
+      }
+    }
   }
 }

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/AlgorithmEvaluator.cs

@@ -64 +64 @@
       var sw = new Stopwatch();
       sw.Start();
-      algorithm.StartSync();
+      algorithm.StartSync(CancellationToken);
       sw.Stop();
       Console.WriteLine("{0},{1}: {2} (Grammar: {3})", ProblemIndexParameter.ActualValue.Value, RepetitionIndexParameter.ActualValue.Value, sw.Elapsed, PrintGrammar(srp.SymbolicExpressionTreeGrammar));

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/AlgorithmRunsAnalyzer.cs

@@ -2 +2 @@
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
 using HeuristicLab.Operators;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using HeuristicLab.Core;
-using HeuristicLab.Common;
-using HeuristicLab.Parameters;
-using HeuristicLab.Data;
-using HeuristicLab.Optimization;
 
 namespace HeuristicLab.Problems.MetaOptimization {
@@ -32 +31 @@
       get { return (ILookupParameter<IItemList<IProblem>>)Parameters[MetaOptimizationProblem.ProblemsParameterName]; }
     }
-    public LookupParameter<DoubleArray> ProblemQualityReferencesParameter {
-      get { return (LookupParameter<DoubleArray>)Parameters["ProblemQualityReferences"]; }
+    public LookupParameter<DoubleArray> ReferenceQualityAveragesParameter {
+      get { return (LookupParameter<DoubleArray>)Parameters["ReferenceQualityAverages"]; }
+    }
+    public LookupParameter<DoubleArray> ReferenceQualityDeviationsParameter {
+      get { return (LookupParameter<DoubleArray>)Parameters["ReferenceQualityDeviations"]; }
+    }
+    public LookupParameter<DoubleArray> ReferenceEvaluatedSolutionAveragesParameter {
+      get { return (LookupParameter<DoubleArray>)Parameters["ReferenceEvaluatedSolutionAverages"]; }
     }
     public LookupParameter<ResultCollection> ResultsParameter {
@@ -47 +52 @@
       get { return (ScopeTreeLookupParameter<IntValue>)Parameters["RepetitionIndex"]; }
     }
+    public LookupParameter<BoolValue> MaximizationParameter {
+      get { return (LookupParameter<BoolValue>)Parameters["Maximization"]; }
+    }
     #endregion
 
@@ -58 +66 @@
       Parameters.Add(new LookupParameter<ParameterConfigurationTree>("ParameterConfigurationTree", ""));
       Parameters.Add(new LookupParameter<IItemList<IProblem>>(MetaOptimizationProblem.ProblemsParameterName, ""));
-      Parameters.Add(new LookupParameter<DoubleArray>("ProblemQualityReferences", ""));
+      Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityAverages", ""));
+      Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityDeviations", ""));
+      Parameters.Add(new LookupParameter<DoubleArray>("ReferenceEvaluatedSolutionAverages", ""));
       Parameters.Add(new LookupParameter<ResultCollection>("Results", ""));
       Parameters.Add(new ScopeTreeLookupParameter<IAlgorithm>("Algorithm", "The finished algorithms containing Runs."));
       Parameters.Add(new ScopeTreeLookupParameter<IntValue>("ProblemIndex", "The index of the problem an algorithm was executed with."));
       Parameters.Add(new ScopeTreeLookupParameter<IntValue>("RepetitionIndex", "The index of the repetition"));
+      Parameters.Add(new LookupParameter<BoolValue>("Maximization", "Set to false if the problem should be minimized."));
     }
     protected AlgorithmRunsAnalyzer(AlgorithmRunsAnalyzer original, Cloner cloner)
@@ -78 +89 @@
       IEnumerable<string> parameterNames = parameterConfiguration.GetOptimizedParameterNames();
       IItemList<IProblem> problems = ProblemsParameter.ActualValue;
+      bool maximization = MaximizationParameter.ActualValue.Value;
       int repetitions = RepetitionsParameter.ActualValue.Value;
-      var resultNames = new List<string> { "BestQuality", "Execution Time" };
+      var resultNames = new List<string> { "BestQuality", "Execution Time", "EvaluatedSolutions" };
       int currentGeneration = GenerationsParameter.ActualValue != null ? GenerationsParameter.ActualValue.Value : 0;
-      double[] referenceQualities = GetReferenceQualities(problems.Count);
+      double[] referenceQualityAverages;
+      double[] referenceQualityDeviations;
+      double[] referenceEvaluatedSolutionAverages;
+      GetReferenceValues(problems.Count, out referenceQualityAverages, out referenceQualityDeviations, out referenceEvaluatedSolutionAverages);
+
       ResultCollection results = ResultsParameter.ActualValue;
 
@@ -88 +104 @@
         var qualities = new double[problems.Count][];
         var executionTimes = new TimeSpan[problems.Count][];
+        var evaluatedSolutions = new int[problems.Count][];
 
         for (int i = 0; i < problems.Count; i++) {
           qualities[i] = new double[repetitions];
+          evaluatedSolutions[i] = new int[repetitions];
           executionTimes[i] = new TimeSpan[repetitions];
         }
@@ -98 +116 @@
           int repetitionIndex = repetitionIndices[i].Value;
 
-          IRun run = algorithms[i].Runs.First();
+          IRun run = algorithms[i].Runs.Single();
           MetaOptimizationUtil.ClearResults(run, resultNames);
           MetaOptimizationUtil.ClearParameters(run, parameterNames);
@@ -107 +125 @@
           qualities[problemIndex][repetitionIndex] = (((DoubleValue)run.Results["BestQuality"]).Value);
           executionTimes[problemIndex][repetitionIndex] = (((TimeSpanValue)run.Results["Execution Time"]).Value);
+          evaluatedSolutions[problemIndex][repetitionIndex] = (((IntValue)run.Results["EvaluatedSolutions"]).Value);
           runs.Add((IRun)run.Clone());
         }
 
         parameterConfiguration.AverageExecutionTimes = new ItemList<TimeSpanValue>(executionTimes.Select(t => new TimeSpanValue(TimeSpan.FromMilliseconds(t.Average(ts => ts.TotalMilliseconds)))));
+        parameterConfiguration.AverageEvaluatedSolutions = new DoubleArray(evaluatedSolutions.Select(x => x.Average()).ToArray());
         parameterConfiguration.Repetitions = new IntValue(repetitions);
-        parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray()); // todo: respect Maximization:true/false
         parameterConfiguration.AverageQualities = new DoubleArray(qualities.Select(q => q.Average()).ToArray());
-        parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray()); // todo: respect Maximization:true/false
+
+        if(maximization)
+          parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray());
+        else
+          parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray());
+        
+        if (maximization)
+          parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray());
+        else
+          parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray());
+
         parameterConfiguration.QualityVariances = new DoubleArray(qualities.Select(q => q.Variance()).ToArray());
         parameterConfiguration.QualityStandardDeviations = new DoubleArray(qualities.Select(q => q.StandardDeviation()).ToArray());
         parameterConfiguration.Runs = runs;
 
-        this.QualityParameter.ActualValue = new DoubleValue(MetaOptimizationUtil.NormalizeQualities(parameterConfiguration, referenceQualities));
+        this.QualityParameter.ActualValue = new DoubleValue(MetaOptimizationUtil.Normalize(parameterConfiguration, referenceQualityAverages, referenceQualityDeviations, referenceEvaluatedSolutionAverages, 1, 1, 1, maximization));
       } else {
         // something terrible happened -> most probably due to invalid parameters.
         // penalty with worst quality from latest generation!
 
-        double penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"]).Value : referenceQualities.Max(); // todo: respect min/max
+        double penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"]).Value : referenceQualityAverages.Max(); // todo: respect min/max
         this.QualityParameter.ActualValue = new DoubleValue(penaltyValue);
       }
@@ -131 +160 @@
     }
 
-    private double[] GetReferenceQualities(int problemsCount) {
-      double[] referenceQualities;
-      if (ProblemQualityReferencesParameter.ActualValue == null) {
+    private void GetReferenceValues(int problemsCount, out double[] referenceQualityAverages, out double[] referenceQualityDeviations, out double[] referenceEvaluatedSolutionAverages) {
+      if (ReferenceQualityAveragesParameter.ActualValue == null) {
         // this is generation zero. no reference qualities for normalization have been calculated yet. in this special case the ReferenceQualityAnalyzer will do the normalization
-        referenceQualities = new double[problemsCount];
-        for (int i = 0; i < referenceQualities.Length; i++) {
-          referenceQualities[i] = 1;
+        referenceQualityAverages = new double[problemsCount];
+        referenceQualityDeviations = new double[problemsCount];
+        referenceEvaluatedSolutionAverages = new double[problemsCount];
+        for (int i = 0; i < referenceQualityAverages.Length; i++) {
+          referenceQualityAverages[i] = 1;
+          referenceQualityDeviations[i] = 1;
+          referenceEvaluatedSolutionAverages[i] = 1;
         }
       } else {
-        referenceQualities = ProblemQualityReferencesParameter.ActualValue.ToArray();
+        referenceQualityAverages = ReferenceQualityAveragesParameter.ActualValue.ToArray();
+        referenceQualityDeviations = ReferenceQualityDeviationsParameter.ActualValue.ToArray();
+        referenceEvaluatedSolutionAverages = ReferenceEvaluatedSolutionAveragesParameter.ActualValue.ToArray();
       }
-      return referenceQualities;
     }
   }

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/AlgorithmSubScopesCreator.cs

@@ -9 +9 @@
 
 namespace HeuristicLab.Problems.MetaOptimization {
-  [Item("ChildrenCreator", "An operator which is used to prepare crossover. The sub-scopes of the current scope the operator is applied on represent the parents. The operator creates new and empty scopes for each child, adds the scopes that represent the child's parents as sub-scopes to the child and adds the child as sub-scope to the current scope.")]
+  [Item("AlgorithmSubScopesCreator", "An operator which ...")]
   [StorableClass]
   public class AlgorithmSubScopesCreator : SingleSuccessorOperator {

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/PMOEvaluator.cs

@@ -1 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
 using HeuristicLab.Operators;
-using HeuristicLab.Common;
-using HeuristicLab.Core;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using HeuristicLab.Parameters;
-using HeuristicLab.Data;
-using HeuristicLab.Optimization;
 
 namespace HeuristicLab.Problems.MetaOptimization {
@@ -34 +30 @@
     public LookupParameter<IntValue> RepetitionsParameter {
       get { return (LookupParameter<IntValue>)Parameters[MetaOptimizationProblem.RepetitionsParameterName]; }
-    }
-    public LookupParameter<DoubleArray> ProblemQualityReferencesParameter {
-      get { return (LookupParameter<DoubleArray>)Parameters["ProblemQualityReferences"]; }
     }
     public LookupParameter<IntValue> GenerationsParameter {
@@ -70 +63 @@
       Parameters.Add(new LookupParameter<ParameterConfigurationTree>("ParameterConfigurationTree", ""));
       Parameters.Add(new LookupParameter<IntValue>(MetaOptimizationProblem.RepetitionsParameterName, "Number of evaluations on one problem."));
-      Parameters.Add(new LookupParameter<DoubleArray>("ProblemQualityReferences", ""));
       Parameters.Add(new LookupParameter<IntValue>("Generations", ""));
       Parameters.Add(new ScopeParameter("CurrentScope", "The current scope which represents a population of solutions on which the genetic algorithm should be applied."));

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/MetaOptimizationUtil.cs

@@ -34 +34 @@
     }
 
-    public static double NormalizeQualities(ParameterConfigurationTree parameterConfigurationTree, double[] referenceQualities) {
-      double[] qualitiesNormalized = new double[referenceQualities.Length];
-      for (int i = 0; i < referenceQualities.Length; i++) {
-        qualitiesNormalized[i] = parameterConfigurationTree.AverageQualities[i] / referenceQualities[i];
+    public static double Normalize(
+          ParameterConfigurationTree parameterConfigurationTree,
+          double[] referenceQualityAverages,
+          double[] referenceQualityDeviations,
+          double[] referenceEvaluatedSolutionAverages,
+          double qualityAveragesWeight,
+          double qualityDeviationsWeight,
+          double evaluatedSolutionsWeight,
+          bool maximization) {
+
+      double[] qualityAveragesNormalized = new double[referenceQualityAverages.Length];
+      double[] qualityDeviationsNormalized = new double[referenceQualityDeviations.Length];
+      double[] evaluatedSolutionAveragesNormalized = new double[referenceEvaluatedSolutionAverages.Length];
+
+      for (int i = 0; i < referenceQualityAverages.Length; i++) {
+        qualityAveragesNormalized[i] = parameterConfigurationTree.AverageQualities[i] / referenceQualityAverages[i];
+        qualityDeviationsNormalized[i] = parameterConfigurationTree.QualityStandardDeviations[i] / referenceQualityDeviations[i];
+        evaluatedSolutionAveragesNormalized[i] = parameterConfigurationTree.AverageEvaluatedSolutions[i] / referenceEvaluatedSolutionAverages[i];
       }
-      parameterConfigurationTree.QualitiesNormalized = new DoubleArray(qualitiesNormalized);
-      parameterConfigurationTree.AverageQualityNormalized = new DoubleValue(qualitiesNormalized.Average());
-      return parameterConfigurationTree.AverageQualityNormalized.Value;
+      parameterConfigurationTree.NormalizedQualityAverages = new DoubleArray(qualityAveragesNormalized);
+      parameterConfigurationTree.NormalizedQualityDeviations = new DoubleArray(qualityDeviationsNormalized);
+      parameterConfigurationTree.NormalizedEvaluatedSolutions = new DoubleArray(evaluatedSolutionAveragesNormalized);
+
+      double qualityAveragesNormalizedValue = qualityAveragesNormalized.Average() * qualityAveragesWeight;
+      double qualityDeviationsNormalizedValue = qualityDeviationsNormalized.Average() * qualityDeviationsWeight;
+      double evaluatedSolutionAveragesNormalizedValue = evaluatedSolutionAveragesNormalized.Average() * evaluatedSolutionsWeight;
+
+      if (maximization) {
+        // deviation and evaluatedSolutions are always minimization problems. so if maximization=true, flip the values around 1.0 (e.g. 1.15 -> 0.85)
+        qualityDeviationsNormalizedValue -= (qualityDeviationsNormalizedValue - 1) * 2;
+        evaluatedSolutionAveragesNormalizedValue -= (evaluatedSolutionAveragesNormalizedValue - 1) * 2;
+      }
+
+      qualityAveragesNormalizedValue *= qualityAveragesWeight;
+      qualityDeviationsNormalizedValue *= qualityDeviationsWeight;
+      evaluatedSolutionAveragesNormalizedValue *= evaluatedSolutionsWeight;
+
+      parameterConfigurationTree.Quality = new DoubleValue(new double[] {qualityAveragesNormalizedValue, qualityDeviationsNormalizedValue, evaluatedSolutionAveragesNormalizedValue}.Average());
+      
+      return parameterConfigurationTree.Quality.Value;
     }
 
@@ -51 +83 @@
       algorithm.Problem = problem;
       ((EngineAlgorithm)algorithm).Engine = new SequentialEngine.SequentialEngine();
-      if(randomize) parameterConfigurationTree.Randomize(random);
+      if (randomize) parameterConfigurationTree.Randomize(random);
       parameterConfigurationTree.Parameterize(algorithm);
       return algorithm;