Changeset 6090 for branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization

Timestamp:

05/01/11 22:02:28 (14 years ago)

Author:

cneumuel

Message:

#1215

• added weight parameters for quality, stddev and evaluated solutions
• lots of fixes

Location:

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3

Files:

• Analyzers/BestParameterConfigurationAnalyzer.cs (modified) (1 diff)
• Analyzers/ReferenceQualityAnalyzer.cs (modified) (4 diffs)
• Analyzers/SolutionCacheAnalyzer.cs (modified) (1 diff)
• Encoding/RangeConstraints/DoubleValueRange.cs (modified) (2 diffs)
• Encoding/RangeConstraints/Range.cs (modified) (4 diffs)
• Encoding/ValueConfigurations/RangeValueConfiguration.cs (modified) (1 diff)
• Evaluators/AlgorithmEvaluator.cs (modified) (5 diffs)
• Evaluators/AlgorithmRunsAnalyzer.cs (modified) (5 diffs)
• MetaOptimizationProblem.cs (modified) (3 diffs)
• MetaOptimizationUtil.cs (modified) (1 diff)

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Analyzers/BestParameterConfigurationAnalyzer.cs

@@ -98 +98 @@
       }
 
-      // population
+      // population (TODO: extract into PopulationAnalyzer)
       int i = 0;
       RunCollection rc = new RunCollection();

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Analyzers/ReferenceQualityAnalyzer.cs

@@ -41 +41 @@
     }
 
+    public LookupParameter<DoubleValue> QualityWeightParameter {
+      get { return (LookupParameter<DoubleValue>)Parameters[MetaOptimizationProblem.QualityWeightParameterName]; }
+    }
+    public LookupParameter<DoubleValue> StandardDeviationWeightParameter {
+      get { return (LookupParameter<DoubleValue>)Parameters[MetaOptimizationProblem.StandardDeviationWeightParameterName]; }
+    }
+    public LookupParameter<DoubleValue> EvaluatedSolutionsWeightParameter {
+      get { return (LookupParameter<DoubleValue>)Parameters[MetaOptimizationProblem.EvaluatedSolutionsWeightParameterName]; }
+    }
+
     public ReferenceQualityAnalyzer()
       : base() {
@@ -49 +59 @@
       Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityDeviations", ""));
       Parameters.Add(new LookupParameter<DoubleArray>("ReferenceEvaluatedSolutionAverages", ""));
-      Parameters.Add(new LookupParameter<ConstrainedItemList<IProblem>>(MetaOptimizationProblem.ProblemsParameterName, ""));
+      Parameters.Add(new LookupParameter<ConstrainedItemList<IProblem>>(MetaOptimizationProblem.ProblemsParameterName));
       Parameters.Add(new LookupParameter<BoolValue>("Maximization", "Set to false if the problem should be minimized."));
+      Parameters.Add(new LookupParameter<DoubleValue>(MetaOptimizationProblem.QualityWeightParameterName));
+      Parameters.Add(new LookupParameter<DoubleValue>(MetaOptimizationProblem.StandardDeviationWeightParameterName));
+      Parameters.Add(new LookupParameter<DoubleValue>(MetaOptimizationProblem.EvaluatedSolutionsWeightParameterName));
     }
 
@@ -65 +78 @@
       ItemArray<DoubleValue> qualities = QualityParameter.ActualValue;
       bool maximization = MaximizationParameter.ActualValue.Value;
+      double qualityWeight = QualityWeightParameter.ActualValue.Value;
+      double standardDeviationWeight = StandardDeviationWeightParameter.ActualValue.Value;
+      double evaluatedSolutionsWeight = EvaluatedSolutionsWeightParameter.ActualValue.Value;
 
       if (ReferenceQualityAveragesParameter.ActualValue == null) {
@@ -76 +92 @@
         ReferenceEvaluatedSolutionAveragesParameter.ActualValue = referenceEvaluatedSolutionAverages;
 
-        NormalizePopulation(solutions, qualities, referenceQualityAverages, referenceQualityDeviations, referenceEvaluatedSolutionAverages, 1, 1, 1, maximization);
+        NormalizePopulation(solutions, qualities, referenceQualityAverages, referenceQualityDeviations, referenceEvaluatedSolutionAverages, qualityWeight, standardDeviationWeight, evaluatedSolutionsWeight, maximization);
 
         results.Add(new Result("ReferenceQualities", referenceQualityAverages));

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Analyzers/SolutionCacheAnalyzer.cs

@@ -71 +71 @@
         }
 
-        var runCollection = allRuns.Single(x => x.Key.Value == key).Value;
-        foreach (var run in solution.Runs) {
-          if (!((BoolValue)run.Results["Meta.FromCache"]).Value || first) {
-            run.Results["Meta.FromCache"] = new BoolValue(true);
-            runCollection.Add(run);
+        if (solution.Runs != null) { // Runs is null when a base-level algorithm exception happened due to invalid parameters
+          var runCollection = allRuns.Single(x => x.Key.Value == key).Value;
+          foreach (var run in solution.Runs) {
+            if (!((BoolValue)run.Results["Meta.FromCache"]).Value || first) {
+              run.Results["Meta.FromCache"] = new BoolValue(true);
+              runCollection.Add(run);
+            }
           }
         }

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Encoding/RangeConstraints/DoubleValueRange.cs

@@ -1 +1 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
 using HeuristicLab.Data;
-using HeuristicLab.Common;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using HeuristicLab.Core;
 
 namespace HeuristicLab.Problems.MetaOptimization {
@@ -12 +10 @@
   public class DoubleValueRange : Range<DoubleValue> {
 
+    public override DoubleValue LowerBound {
+      get {
+        return base.LowerBound;
+      }
+      set {
+        base.LowerBound = value;
+      }
+    }
+    
     public DoubleValueRange(DoubleValue lowerBound, DoubleValue upperBound, DoubleValue stepSize) : base(lowerBound, upperBound, stepSize) { }
     public DoubleValueRange() { }

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Encoding/RangeConstraints/Range.cs

@@ -12 +12 @@
     [Storable]
     private T lowerBound;
-    public T LowerBound {
+    public virtual T LowerBound {
       get { return lowerBound; }
       set {
@@ -30 +30 @@
     [Storable]
     private T upperBound;
-    public T UpperBound {
+    public virtual T UpperBound {
       get { return upperBound; }
       set {
@@ -48 +48 @@
     [Storable]
     private T stepSize;
-    public T StepSize {
+    public virtual T StepSize {
       get { return stepSize; }
       set {
@@ -197 +197 @@
       if (random.NextDouble() < 0.1) {
         if (random.NextDouble() < 0.5) {
-          return LowerBound;
+          return (T)LowerBound.Clone();
         } else {
-          return UpperBound;
+          return (T)UpperBound.Clone();
         }
       }

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Encoding/ValueConfigurations/RangeValueConfiguration.cs

@@ -36 +36 @@
       : base(value, valueDataType) {
       if (actualValue.ValueDataType == typeof(IntValue)) {
-        RangeConstraint = new IntValueRange(new IntValue(0), (IntValue)value, new IntValue(1));
+        RangeConstraint = new IntValueRange(new IntValue(0), (IntValue)value.Clone(), new IntValue(1));
       } else if (actualValue.ValueDataType == typeof(DoubleValue)) {
-        RangeConstraint = new DoubleValueRange(new DoubleValue(0), (DoubleValue)value, new DoubleValue(0.01));
+        RangeConstraint = new DoubleValueRange(new DoubleValue(0), (DoubleValue)value.Clone(), new DoubleValue(0.01));
       } else if (actualValue.ValueDataType == typeof(PercentValue)) {
         RangeConstraint = new PercentValueRange(new PercentValue(0), new PercentValue(1), new PercentValue(0.01));

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/AlgorithmEvaluator.cs

@@ -1 @@
-using System;
-using System.Diagnostics;
-using System.Linq;
+using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -10 +8 @@
 using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
 
 namespace HeuristicLab.Problems.MetaOptimization {
@@ -33 +30 @@
       get { return (ILookupParameter<ParameterConfigurationTree>)Parameters["ParameterConfigurationTree"]; }
     }
-    public IValueParameter<IItemList<IRun>> RunsParameter {
-      get { return (IValueParameter<IItemList<IRun>>)Parameters["Runs"]; }
+    //public IValueParameter<IItemList<IRun>> RunsParameter {
+    //  get { return (IValueParameter<IItemList<IRun>>)Parameters["Runs"]; }
+    //}
+    public LookupParameter<ResultCollection> ResultsParameter {
+      get { return (LookupParameter<ResultCollection>)Parameters["Results"]; }
     }
     #endregion
@@ -47 +47 @@
       Parameters.Add(new LookupParameter<IntValue>("RepetitionIndex", ""));
       Parameters.Add(new LookupParameter<ParameterConfigurationTree>("ParameterConfigurationTree", ""));
-      Parameters.Add(new ValueParameter<IItemList<IRun>>("Runs", ""));
+      Parameters.Add(new LookupParameter<ResultCollection>("Results", ""));
     }
     protected AlgorithmEvaluator(AlgorithmEvaluator original, Cloner cloner) : base(original, cloner) { }
@@ -55 +55 @@
 
     public override IOperation Apply() {
+      ItemDictionary<StringValue, RunCollection> solutionCache = ResultsParameter.ActualValue.ContainsKey("SolutionCache") ? (ItemDictionary<StringValue, RunCollection>)ResultsParameter.ActualValue["SolutionCache"].Value : null;
+      ParameterConfigurationTree parameterConfiguration = ParameterConfigurationParameter.ActualValue;
       IAlgorithm algorithm = AlgorithmParameter.ActualValue;
+      int repetitionIndex = RepetitionIndexParameter.ActualValue.Value;
 
-      #region Debug Code for SymbolicRegressionGrammar
-      var srp = (SymbolicRegressionSingleObjectiveProblem)algorithm.Problem;
-      //Console.WriteLine("Evaluating Grammar: " + PrintGrammar(srp.SymbolicExpressionTreeGrammar));
-      #endregion
-
-      var sw = new Stopwatch();
-      sw.Start();
-      //// prepare and clear is needed, if the algorithm has been started and stopped before (this happens when meta level algorithm is paused)
-      //algorithm.Prepare(); // <--- SHOULD NOT HAPPEN!
-      //algorithm.Runs.Clear();
-
-      algorithm.StartSync(CancellationToken);
-      sw.Stop();
-      Console.WriteLine("{0},{1}: {2} (Grammar: {3})", ProblemIndexParameter.ActualValue.Value, RepetitionIndexParameter.ActualValue.Value, sw.Elapsed, PrintGrammar(srp.SymbolicExpressionTreeGrammar));
-
-      RunsParameter.Value = new ItemList<IRun>(algorithm.Runs);
-
+      if (solutionCache != null && solutionCache.Count(x => x.Key.Value == parameterConfiguration.ParameterInfoString) > 0) {
+        algorithm.Runs.Add(solutionCache.Single(x => x.Key.Value == parameterConfiguration.ParameterInfoString).Value.ElementAt(repetitionIndex));
+      } else {
+        algorithm.StartSync(CancellationToken);
+      }
       return base.Apply();
-    }
-
-    /// <summary>
-    /// This method should repair an invalid parameterConfiguration.
-    /// The strategy is to just randomize parameter settings. It's not guaranteed to be a vaild setting
-    /// </summary>
-    private void Repair(ParameterConfigurationTree parameterConfiguration, IRandom random) {
-      parameterConfiguration.Randomize(random);
     }
 

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/AlgorithmRunsAnalyzer.cs

@@ -54 +54 @@
     public LookupParameter<BoolValue> MaximizationParameter {
       get { return (LookupParameter<BoolValue>)Parameters["Maximization"]; }
+    }
+    public LookupParameter<DoubleValue> QualityWeightParameter {
+      get { return (LookupParameter<DoubleValue>)Parameters[MetaOptimizationProblem.QualityWeightParameterName]; }
+    }
+    public LookupParameter<DoubleValue> StandardDeviationWeightParameter {
+      get { return (LookupParameter<DoubleValue>)Parameters[MetaOptimizationProblem.StandardDeviationWeightParameterName]; }
+    }
+    public LookupParameter<DoubleValue> EvaluatedSolutionsWeightParameter {
+      get { return (LookupParameter<DoubleValue>)Parameters[MetaOptimizationProblem.EvaluatedSolutionsWeightParameterName]; }
     }
     #endregion
@@ -75 +84 @@
       Parameters.Add(new ScopeTreeLookupParameter<IntValue>("RepetitionIndex", "The index of the repetition"));
       Parameters.Add(new LookupParameter<BoolValue>("Maximization", "Set to false if the problem should be minimized."));
+      Parameters.Add(new LookupParameter<DoubleValue>(MetaOptimizationProblem.QualityWeightParameterName));
+      Parameters.Add(new LookupParameter<DoubleValue>(MetaOptimizationProblem.StandardDeviationWeightParameterName));
+      Parameters.Add(new LookupParameter<DoubleValue>(MetaOptimizationProblem.EvaluatedSolutionsWeightParameterName));
     }
     protected AlgorithmRunsAnalyzer(AlgorithmRunsAnalyzer original, Cloner cloner)
@@ -92 +104 @@
       bool maximization = MaximizationParameter.ActualValue.Value;
       int repetitions = RepetitionsParameter.ActualValue.Value;
+      double qualityWeight = QualityWeightParameter.ActualValue.Value;
+      double standardDeviationWeight = StandardDeviationWeightParameter.ActualValue.Value;
+      double evaluatedSolutionsWeight = EvaluatedSolutionsWeightParameter.ActualValue.Value;
       var resultNames = new List<string> { "BestQuality", "Execution Time", "EvaluatedSolutions" };
       int currentGeneration = GenerationsParameter.ActualValue != null ? GenerationsParameter.ActualValue.Value : 0;
@@ -149 +164 @@
         parameterConfiguration.Runs = runs;
 
-        this.QualityParameter.ActualValue = new DoubleValue(MetaOptimizationUtil.Normalize(parameterConfiguration, referenceQualityAverages, referenceQualityDeviations, referenceEvaluatedSolutionAverages, 1, 0.1, 1, maximization));
+        this.QualityParameter.ActualValue = new DoubleValue(MetaOptimizationUtil.Normalize(parameterConfiguration, referenceQualityAverages, referenceQualityDeviations, referenceEvaluatedSolutionAverages, qualityWeight, standardDeviationWeight, evaluatedSolutionsWeight, maximization));
       } else {
         // something terrible happened -> most probably due to invalid parameters.
@@ -155 +170 @@
         double penaltyValue;
         if (maximization)
-          penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"]).Value : referenceQualityAverages.Min();
+          penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"].Value).Value : referenceQualityAverages.Min();
         else
-          penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"]).Value : referenceQualityAverages.Max();
+          penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"].Value).Value : referenceQualityAverages.Max();
+        
         this.QualityParameter.ActualValue = new DoubleValue(penaltyValue);
+        parameterConfiguration.Quality = new DoubleValue(penaltyValue);
+
+        parameterConfiguration.AverageExecutionTimes = new ItemList<TimeSpanValue>(Enumerable.Repeat(new TimeSpanValue(TimeSpan.Zero), problems.Count));
+        parameterConfiguration.AverageEvaluatedSolutions = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
+        parameterConfiguration.Repetitions = new IntValue(repetitions);
+        parameterConfiguration.AverageQualities = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
+        parameterConfiguration.BestQualities = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
+        parameterConfiguration.WorstQualities = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
+        parameterConfiguration.QualityVariances = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
+        parameterConfiguration.QualityStandardDeviations = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
+        parameterConfiguration.Runs = null;
       }
 

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/MetaOptimizationProblem.cs

@@ -51 +51 @@
     public const string IntValueCrossoverParameterName = "IntValueCrossover";
     public const string DoubleValueCrossoverParameterName = "DoubleValueCrossover";
+    public const string QualityWeightParameterName = "QualityWeight";
+    public const string StandardDeviationWeightParameterName = "StandardDeviationWeight";
+    public const string EvaluatedSolutionsWeightParameterName = "EvaluatedSolutionsWeight";
+
 
     #region Parameter Properties
@@ -134 +138 @@
       var validIntCrossovers = new ItemSet<IIntValueCrossover>(ApplicationManager.Manager.GetInstances<IIntValueCrossover>());
       var validDoubleCrossovers = new ItemSet<IDoubleValueCrossover>(ApplicationManager.Manager.GetInstances<IDoubleValueCrossover>());
-      Parameters.Add(new ConstrainedValueParameter<IIntValueManipulator>(IntValueManipulatorParameterName, "", validIntManipulators, validIntManipulators.Where(x => x.GetType() == typeof(NormalIntValueManipulator)).SingleOrDefault()));
-      Parameters.Add(new ConstrainedValueParameter<IDoubleValueManipulator>(DoubleValueManipulatorParameterName, "", validDoubleManipulators, validDoubleManipulators.Where(x => x.GetType() == typeof(NormalDoubleValueManipulator)).SingleOrDefault()));
-      Parameters.Add(new ConstrainedValueParameter<IIntValueCrossover>(IntValueCrossoverParameterName, "", validIntCrossovers, validIntCrossovers.Where(x => x.GetType() == typeof(NormalIntValueCrossover)).SingleOrDefault()));
-      Parameters.Add(new ConstrainedValueParameter<IDoubleValueCrossover>(DoubleValueCrossoverParameterName, "", validDoubleCrossovers, validDoubleCrossovers.Where(x => x.GetType() == typeof(NormalDoubleValueCrossover)).SingleOrDefault()));
+      Parameters.Add(new ConstrainedValueParameter<IIntValueManipulator>(IntValueManipulatorParameterName, validIntManipulators, validIntManipulators.Where(x => x.GetType() == typeof(NormalIntValueManipulator)).SingleOrDefault()));
+      Parameters.Add(new ConstrainedValueParameter<IDoubleValueManipulator>(DoubleValueManipulatorParameterName, validDoubleManipulators, validDoubleManipulators.Where(x => x.GetType() == typeof(NormalDoubleValueManipulator)).SingleOrDefault()));
+      Parameters.Add(new ConstrainedValueParameter<IIntValueCrossover>(IntValueCrossoverParameterName, validIntCrossovers, validIntCrossovers.Where(x => x.GetType() == typeof(NormalIntValueCrossover)).SingleOrDefault()));
+      Parameters.Add(new ConstrainedValueParameter<IDoubleValueCrossover>(DoubleValueCrossoverParameterName, validDoubleCrossovers, validDoubleCrossovers.Where(x => x.GetType() == typeof(NormalDoubleValueCrossover)).SingleOrDefault()));
+
+      Parameters.Add(new ValueParameter<DoubleValue>(QualityWeightParameterName, new DoubleValue(1)));
+      Parameters.Add(new ValueParameter<DoubleValue>(StandardDeviationWeightParameterName, new DoubleValue(0.01)));
+      Parameters.Add(new ValueParameter<DoubleValue>(EvaluatedSolutionsWeightParameterName, new DoubleValue(0.0005)));
 
       Maximization = new BoolValue(false);
@@ -182 +190 @@
     private void InitializeOperators() {
       Operators.AddRange(ApplicationManager.Manager.GetInstances<IParameterConfigurationOperator>().Cast<IOperator>());
+      Operators.Add(new ReferenceQualityAnalyzer());
       Operators.Add(new BestParameterConfigurationAnalyzer());
-      Operators.Add(new ReferenceQualityAnalyzer());
       Operators.Add(new SolutionCacheAnalyzer());
       Operators.Add(new PMOPopulationDiversityAnalyzer());

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/MetaOptimizationUtil.cs

@@ -57 +57 @@
       parameterConfigurationTree.NormalizedEvaluatedSolutions = new DoubleArray(evaluatedSolutionAveragesNormalized);
 
-      double qualityAveragesNormalizedValue = qualityAveragesNormalized.Average() * qualityAveragesWeight;
-      double qualityDeviationsNormalizedValue = qualityDeviationsNormalized.Average() * qualityDeviationsWeight;
-      double evaluatedSolutionAveragesNormalizedValue = evaluatedSolutionAveragesNormalized.Average() * evaluatedSolutionsWeight;
+      double qualityAveragesNormalizedValue = qualityAveragesNormalized.Average();
+      double qualityDeviationsNormalizedValue = qualityDeviationsNormalized.Average();
+      double evaluatedSolutionAveragesNormalizedValue = evaluatedSolutionAveragesNormalized.Average();
 
+      // deviation and evaluatedSolutions are always minimization problems. so if maximization=true, flip the values around 1.0 (e.g. 1.15 -> 0.85)
       if (maximization) {
-        // deviation and evaluatedSolutions are always minimization problems. so if maximization=true, flip the values around 1.0 (e.g. 1.15 -> 0.85)
         qualityDeviationsNormalizedValue -= (qualityDeviationsNormalizedValue - 1) * 2;
         evaluatedSolutionAveragesNormalizedValue -= (evaluatedSolutionAveragesNormalizedValue - 1) * 2;
       }
 
+      // apply weights
       qualityAveragesNormalizedValue *= qualityAveragesWeight;
       qualityDeviationsNormalizedValue *= qualityDeviationsWeight;
       evaluatedSolutionAveragesNormalizedValue *= evaluatedSolutionsWeight;
 
-      parameterConfigurationTree.Quality = new DoubleValue(new double[] {qualityAveragesNormalizedValue, qualityDeviationsNormalizedValue, evaluatedSolutionAveragesNormalizedValue}.Average());
-      
+      double weightSum = qualityAveragesWeight + qualityDeviationsWeight + evaluatedSolutionsWeight;
+      parameterConfigurationTree.Quality = new DoubleValue((qualityAveragesNormalizedValue + qualityDeviationsNormalizedValue + evaluatedSolutionAveragesNormalizedValue) / weightSum);
+
       return parameterConfigurationTree.Quality.Value;
     }