Changeset 5303 for branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization

Timestamp:

01/17/11 01:04:30 (14 years ago)

Author:

cneumuel

Message:

#1215

• manipulators for one and all parameters
• SolutionCache to avoid multiple evaluations of equal solutions
• RunsAnalyzer which stores all base level runs
• ItemDictionaryView for runs

Location:

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3

Files:

• Analyzers/RunsAnalyzer.cs (added)
• Encoding/Manipulators/ParameterConfigurationAllPositionsManipulator.cs (added)
• Encoding/Manipulators/ParameterConfigurationManipulator.cs (modified) (3 diffs)
• Encoding/Manipulators/ParameterConfigurationOnePositionsManipulator.cs (added)
• Encoding/ParameterConfigurationTree.cs (modified) (1 diff)
• Encoding/ParameterConfigurations/ParameterConfiguration.cs (modified) (1 diff)
• Encoding/ValueConfigurations/ValueConfiguration.cs (modified) (1 diff)
• Evaluators/ParameterConfigurationEvaluator.cs (modified) (4 diffs)
• HeuristicLab.Problems.MetaOptimization-3.3.csproj (modified) (2 diffs)
• Interfaces/IOptimizable.cs (modified) (1 diff)
• MetaOptimizationProblem.cs (modified) (5 diffs)

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Encoding/Manipulators/ParameterConfigurationManipulator.cs

@@ -14 +14 @@
   // todo: item name/descr...
   [StorableClass]
-  public class ParameterConfigurationManipulator : SingleSuccessorOperator, IParameterConfigurationManipulator, IStochasticOperator {
+  public abstract class ParameterConfigurationManipulator : SingleSuccessorOperator, IParameterConfigurationManipulator, IStochasticOperator {
     public override bool CanChangeName {
       get { return false; }
@@ -44 +44 @@
       : base(original, cloner) {
     }
-    public override IDeepCloneable Clone(Cloner cloner) {
-      return new ParameterConfigurationManipulator(this, cloner);
-    }
 
-    public sealed override IOperation Apply() {
+    public override IOperation Apply() {
       Apply(RandomParameter.ActualValue, ParameterConfigurationTreeParameter.ActualValue, IntValueManipulatorParameter.ActualValue, DoubleValueManipulatorParameter.ActualValue);
       return base.Apply();
@@ -57 +54 @@
     }
 
-    private static void Mutate(IRandom random, IOptimizable configuration, IIntValueManipulator intValueManipulator, IDoubleValueManipulator doubleValueManipulator) {
+    protected static void Mutate(IRandom random, IOptimizable configuration, IIntValueManipulator intValueManipulator, IDoubleValueManipulator doubleValueManipulator) {
       var vc = configuration as IValueConfiguration;
       var pc = configuration as IParameterConfiguration;

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Encoding/ParameterConfigurationTree.cs

@@ -213 +213 @@
       return cnt;
     }
+
+    public IOptimizable GetRandomOptimizable(IRandom random) {
+      List<IOptimizable> allOptimizables = GetAllOptimizables();
+      return allOptimizables[random.Next(allOptimizables.Count)];
+    }
   }
 }

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Encoding/ParameterConfigurations/ParameterConfiguration.cs

@@ -455 +455 @@
     }
 
+    public List<IOptimizable> GetAllOptimizables() {
+      var list = new List<IOptimizable>();
+      foreach (var vc in ValueConfigurations) {
+        if (vc.Optimize) {
+          list.Add(vc);
+          list.AddRange(vc.GetAllOptimizables());
+        }
+      }
+      return list;
+    }
   }
 }

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Encoding/ValueConfigurations/ValueConfiguration.cs

@@ -269 +269 @@
       }
     }
+
+    public List<IOptimizable> GetAllOptimizables() {
+      var list = new List<IOptimizable>();
+      foreach (var pc in ParameterConfigurations) {
+        if(pc.Optimize) {
+          if(pc.ValueConfigurations.CheckedItems.Count() > 1) list.Add(pc); // only add if there are more than 1 choices. otherwise it makes no sense to optimize which VC is selected
+          list.AddRange(pc.GetAllOptimizables());
+        }
+      }
+      return list;
+    }
   }
 }

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/ParameterConfigurationEvaluator.cs

@@ -43 +43 @@
       get { return (LookupParameter<DoubleArray>)Parameters["ProblemQualityReferences"]; }
     }
-
+    public LookupParameter<IntValue> GenerationsParameter {
+      get { return (LookupParameter<IntValue>)Parameters["Generations"]; }
+    }
+    public LookupParameter<ResultCollection> ResultsParameter {
+      get { return (LookupParameter<ResultCollection>)Parameters["Results"]; }
+    }
+    
     public IntValue Repetitions {
       get { return RepetitionsParameter.ActualValue; }
@@ -56 +62 @@
       Parameters.Add(new LookupParameter<IntValue>(MetaOptimizationProblem.RepetitionsParameterName, "Number of evaluations on one problem."));
       Parameters.Add(new LookupParameter<DoubleArray>("ProblemQualityReferences", ""));
+      Parameters.Add(new LookupParameter<IntValue>("Generations", ""));
+      Parameters.Add(new LookupParameter<ResultCollection>("Results", ""));
     }
 
@@ -68 +76 @@
 
     public override IOperation Apply() {
-      EngineAlgorithm algorithm = (EngineAlgorithm)AlgorithmParameter.ActualValue.Clone();
+      ParameterConfigurationTree parameterConfiguration = ParameterConfigurationParameter.ActualValue;
+      EngineAlgorithm algorithm = (EngineAlgorithm)AlgorithmParameter.ActualValue;
       IItemList<ISingleObjectiveProblem> problems = ProblemsParameter.ActualValue;
-      ParameterConfigurationTree parameterConfiguration = ParameterConfigurationParameter.ActualValue;
+      ItemDictionary<StringValue, RunCollection> runsCache = ResultsParameter.ActualValue.ContainsKey("Runs") ? (ItemDictionary<StringValue, RunCollection>)ResultsParameter.ActualValue["Runs"].Value : null;
+      double[] referenceQualities = GetReferenceQualities(problems);
 
-      double[] referenceQualities;
-      if (ProblemQualityReferencesParameter.ActualValue == null) {
-        // this is generation zero. no reference qualities for normalization have been calculated yet.
-        referenceQualities = new double[problems.Count];
-        for (int i = 0; i < referenceQualities.Length; i++) {
-          referenceQualities[i] = 1;
-        }
+      RunCollection runs;
+      if (runsCache != null && runsCache.Count(x => x.Key.Value == parameterConfiguration.ParameterInfoString) > 0) {
+        runs = runsCache.Single(x => x.Key.Value == parameterConfiguration.ParameterInfoString).Value;
+        Console.WriteLine("Used Cache for {0}", parameterConfiguration.ParameterInfoString);
       } else {
-        referenceQualities = ProblemQualityReferencesParameter.ActualValue.ToArray();
+        runs = ExecuteAlgorithm(parameterConfiguration, algorithm, problems);
       }
-
-      // set parameters
-      parameterConfiguration.Parameterize(algorithm);
-      algorithm.StoreAlgorithmInEachRun = false;
 
       List<List<double>> qualities = new List<List<double>>();
       List<List<TimeSpan>> executionTimes = new List<List<TimeSpan>>();
-      algorithm.Engine = new SequentialEngine.SequentialEngine();
-      algorithm.Prepare(true);
 
-      foreach (ISingleObjectiveProblem problem in problems) {
-        algorithm.Problem = (IProblem)problem.Clone();
-        var problemQualities = new List<double>();
-        var problemExecutionTimes = new List<TimeSpan>();
-
-        for (int i = 0; i < Repetitions.Value; i++) {
-          algorithm.Prepare();
-
-          AlgorithmExecutor executor = new AlgorithmExecutor(algorithm);
-          executor.StartSync();
-
-          if (algorithm.ExecutionState == ExecutionState.Paused) {
-            // this parametercombination was bad. set penalty for this solution
-            problemQualities.Add(PenaltyQuality); // todo: respect Maximization; problem: this messes up average value; solution: use currently worst quality*2
-            problemExecutionTimes.Add(algorithm.ExecutionTime);
-          } else {
-            problemQualities.Add(((DoubleValue)algorithm.Results["BestQuality"].Value).Value);
-            problemExecutionTimes.Add(algorithm.ExecutionTime);
-
-            // parameters will be stored in ParameterConfigurationTree anyway. they would be redundant in runs
-            algorithm.Runs.Last().Parameters.Clear();
-            // but keep the problem, since this differs in runs
-            algorithm.Runs.Last().Parameters.Add("Problem", problem);
-          }
-        }
-        qualities.Add(problemQualities);
-        executionTimes.Add(problemExecutionTimes);
+      for (int i = 0; i < problems.Count; i++) {
+        var problemRuns = runs.Where(x => ((IntValue)x.Results["Meta.ProblemIndex"]).Value == i + 1);
+        qualities.Add(problemRuns.Select(x => ((DoubleValue)x.Results["BestQuality"]).Value).ToList());
+        executionTimes.Add(problemRuns.Select(x => ((TimeSpanValue)x.Results["Execution Time"]).Value).ToList());
       }
-      algorithm.Prepare();
 
       parameterConfiguration.AverageExecutionTimes = new ItemList<TimeSpanValue>(executionTimes.Select(t => new TimeSpanValue(TimeSpan.FromMilliseconds(t.Average(ts => ts.TotalMilliseconds)))));
@@ -129 +106 @@
       parameterConfiguration.QualityVariances = new DoubleArray(qualities.Select(q => q.Variance()).ToArray());
       parameterConfiguration.QualityStandardDeviations = new DoubleArray(qualities.Select(q => q.StandardDeviation()).ToArray());
-      parameterConfiguration.Runs = (RunCollection)algorithm.Runs.Clone();
+      parameterConfiguration.Runs = (RunCollection)runs.Clone();
 
       this.QualityParameter.ActualValue = new DoubleValue(NormalizeQualities(parameterConfiguration, referenceQualities));
 
       return base.Apply();
+    }
+
+    private double[] GetReferenceQualities(IItemList<ISingleObjectiveProblem> problems) {
+      double[] referenceQualities;
+      if (ProblemQualityReferencesParameter.ActualValue == null) {
+        // this is generation zero. no reference qualities for normalization have been calculated yet. in this special case the ReferenceQualityAnalyzer will do the normalization
+        referenceQualities = new double[problems.Count];
+        for (int i = 0; i < referenceQualities.Length; i++) {
+          referenceQualities[i] = 1;
+        }
+      } else {
+        referenceQualities = ProblemQualityReferencesParameter.ActualValue.ToArray();
+      }
+      return referenceQualities;
+    }
+
+    private RunCollection ExecuteAlgorithm(ParameterConfigurationTree parameterConfiguration, EngineAlgorithm algorithm, IItemList<ISingleObjectiveProblem> problems) {
+      EngineAlgorithm algorithmClone = (EngineAlgorithm)algorithm.Clone();
+
+      // set parameters
+      parameterConfiguration.Parameterize(algorithmClone);
+      algorithmClone.StoreAlgorithmInEachRun = false;
+
+      algorithmClone.Engine = new SequentialEngine.SequentialEngine();
+      algorithmClone.Prepare(true);
+
+      foreach (ISingleObjectiveProblem problem in problems) {
+        algorithmClone.Problem = (IProblem)problem.Clone();
+
+        for (int i = 0; i < Repetitions.Value; i++) {
+          algorithmClone.Prepare();
+
+          AlgorithmExecutor executor = new AlgorithmExecutor(algorithmClone);
+          executor.StartSync();
+
+          if (algorithmClone.ExecutionState == ExecutionState.Paused) {
+            // this parametercombination was bad. set penalty for this solution
+            // TODO!
+          }
+          int problemIndex = problems.IndexOf(problem) + 1;
+          IRun run = algorithmClone.Runs.Last();
+          run.Results.Add("Meta.FromCache", new BoolValue(false));
+          run.Results.Add("Meta.Generation", new IntValue(GenerationsParameter.ActualValue != null ? GenerationsParameter.ActualValue.Value : 0));
+          run.Results.Add("Meta.ProblemIndex", new IntValue(problemIndex));
+          int runCountOfThisProblem = algorithmClone.Runs.Where(x => ((IntValue)x.Results["Meta.ProblemIndex"]).Value == problemIndex).Count();
+          run.Name = string.Format("{0} Problem {1} Run {2}", parameterConfiguration.ParameterInfoString, problemIndex, runCountOfThisProblem);
+        }
+      }
+      algorithmClone.Prepare();
+      return algorithmClone.Runs;
     }
 

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/HeuristicLab.Problems.MetaOptimization-3.3.csproj

@@ -135 +135 @@
     <None Include="Properties\AssemblyInfo.cs.frame" />
     <None Include="HeuristicLabProblemsMetaOptimizationPlugin.cs.frame" />
+    <Compile Include="Analyzers\RunsAnalyzer.cs" />
     <Compile Include="Analyzers\ReferenceQualityAnalyzer.cs" />
     <Compile Include="ConstrainedItemList.cs" />
+    <Compile Include="Encoding\Manipulators\ParameterConfigurationManipulator.cs" />
+    <Compile Include="Encoding\Manipulators\ParameterConfigurationOnePositionsManipulator.cs" />
     <Compile Include="Encoding\NullValue.cs" />
     <Compile Include="Encoding\ParameterCombinationsEnumerator.cs" />
@@ -176 +179 @@
     <Compile Include="Encoding\RangeConstraints\Range.cs" />
     <Compile Include="Interfaces\IParameterConfigurationManipulator.cs" />
-    <Compile Include="Encoding\Manipulators\ParameterConfigurationManipulator.cs" />
+    <Compile Include="Encoding\Manipulators\ParameterConfigurationAllPositionsManipulator.cs" />
     <Compile Include="Operators\Interfaces\IIntValueCrossover.cs" />
     <Compile Include="Operators\Interfaces\IValueManipulator.cs" />

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Interfaces/IOptimizable.cs

@@ -20 +20 @@
     string ParameterInfoString { get; }
 
+    /// <summary>
+    /// Recursively gets all optimizables which have Optimize=true
+    /// </summary>
+    List<IOptimizable> GetAllOptimizables();
+
     event EventHandler IsOptimizableChanged;
     event EventHandler OptimizeChanged;

branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/MetaOptimizationProblem.cs

@@ -101 +101 @@
       get { return Operators.OfType<ReferenceQualityAnalyzer>().FirstOrDefault(); }
     }
-    #endregion
-
-    public MetaOptimizationProblem() : base() {
+    private RunsAnalyzer RunsAnalyzer {
+      get { return Operators.OfType<RunsAnalyzer>().FirstOrDefault(); }
+    }
+    #endregion
+
+    public MetaOptimizationProblem()
+      : base() {
       Parameters.Add(new ValueParameter<EngineAlgorithm>(AlgorithmTypeParameterName, "The algorithm which's parameters should be optimized.", new GeneticAlgorithm()));
       Parameters.Add(new ValueParameter<ISingleObjectiveProblem>(ProblemTypeParameterName, "The problem type.", new SingleObjectiveTestFunctionProblem()));
@@ -110 +114 @@
       Parameters.Add(new ValueParameter<IntValue>(RepetitionsParameterName, "The number of evaluations for each problem.", new IntValue(3)));
 
-      var validIntManipulators = new ItemSet<IIntValueManipulator>( ApplicationManager.Manager.GetInstances<IIntValueManipulator>());
+      var validIntManipulators = new ItemSet<IIntValueManipulator>(ApplicationManager.Manager.GetInstances<IIntValueManipulator>());
       var validDoubleManipulators = new ItemSet<IDoubleValueManipulator>(ApplicationManager.Manager.GetInstances<IDoubleValueManipulator>());
       var validIntCrossovers = new ItemSet<IIntValueCrossover>(ApplicationManager.Manager.GetInstances<IIntValueCrossover>());
@@ -137 +141 @@
     [StorableConstructor]
     private MetaOptimizationProblem(bool deserializing) : base(deserializing) { }
-    private MetaOptimizationProblem(MetaOptimizationProblem original, Cloner cloner) : base(original, cloner) {
+    private MetaOptimizationProblem(MetaOptimizationProblem original, Cloner cloner)
+      : base(original, cloner) {
       // todo
       this.RegisterParameterEvents();
@@ -157 +162 @@
       ProblemTypeParameter.ValueChanged += new EventHandler(ProblemTypeParameter_ValueChanged);
     }
-    
+
     private void InitializeOperators() {
       Operators.AddRange(ApplicationManager.Manager.GetInstances<IParameterConfigurationOperator>().Cast<IOperator>());
       Operators.Add(new BestParameterConfigurationAnalyzer());
       Operators.Add(new ReferenceQualityAnalyzer());
+      Operators.Add(new RunsAnalyzer());
     }
     private void ParameterizeSolutionCreator() {
@@ -174 +180 @@
       if (ReferenceQualityAnalyzer != null) {
         ReferenceQualityAnalyzer.ParameterConfigurationParameter.ActualName = ((RandomParameterConfigurationCreator)SolutionCreator).ParameterConfigurationParameter.ActualName;
+      }
+      if (RunsAnalyzer != null) {
+        RunsAnalyzer.ParameterConfigurationParameter.ActualName = ((RandomParameterConfigurationCreator)SolutionCreator).ParameterConfigurationParameter.ActualName;
       }
     }