Changeset 12804 for branches/PerformanceComparison/HeuristicLab.Analysis/3.3/Optimizers

Timestamp:

07/26/15 21:32:52 (9 years ago)

Author:

abeham

Message:

#2431:

• worked on IRRRun (early abort still troublesome)
• Updated RLD view to allow defining targets
• Attempting to handle maximization/minimization

File:

• branches/PerformanceComparison/HeuristicLab.Analysis/3.3/Optimizers/IRRestarter.cs (moved) (moved from branches/PerformanceComparison/HeuristicLab.Analysis/3.3/Optimizers/IRRRun.cs) (16 diffs)

branches/PerformanceComparison/HeuristicLab.Analysis/3.3/Optimizers/IRRestarter.cs

@@ -37 +37 @@
   /// A run in which an algorithm is executed for a certain maximum time only.
   /// </summary>
-  [Item("Independent Random Restart Run", "A run in which an optimizer is repeated until either a certain target value is reached or a maximum budget is exceeded.")]
+  [Item("Independent Random Restarter", "An optimizer that repeats an algorithm until either a certain target value is reached or a maximum budget is exceeded.")]
   [Creatable(CreatableAttribute.Categories.TestingAndAnalysis, Priority = 117)]
   [StorableClass]
-  public sealed class IndepdentRandomRestartRun : NamedItem, IOptimizer, IStorableContent, INotifyPropertyChanged {
+  public sealed class IndepdentRandomRestarter : NamedItem, IOptimizer, IStorableContent, INotifyPropertyChanged {
+    private const string ExecutionTimeResultName = "Execution Time";
+    private const string BestQualityResultName = "BestQuality";
+
     public string Filename { get; set; }
 
@@ -100 +103 @@
     public bool Maximization {
       get { return maximization; }
-      private set {
+      set {
         if (maximization == value) return;
         maximization = value;
@@ -114 +117 @@
         if (moveCostPerSolution == value) return;
         moveCostPerSolution = value;
+        perEvaluationsAnalyzer.MoveCostPerSolutionParameter.Value = new DoubleValue(moveCostPerSolution);
         OnPropertyChanged("MoveCostPerSolution");
       }
     }
 
-
+    [Storable]
+    private QualityPerClockAnalyzer perClockAnalyzer;
+    [Storable]
+    private QualityPerEvaluationsAnalyzer perEvaluationsAnalyzer;
+
+    [Storable]
+    private ExecutionState executionState;
     public ExecutionState ExecutionState {
-      get { return (Algorithm != null) ? Algorithm.ExecutionState : ExecutionState.Stopped; }
+      get { return executionState; }
+      private set {
+        if (executionState != value) {
+          executionState = value;
+          OnExecutionStateChanged();
+          OnItemImageChanged();
+        }
+      }
     }
 
@@ -176 +193 @@
       get { return algorithm; }
       set {
+        if (value != null && !typeof(ISingleObjectiveHeuristicOptimizationProblem).IsAssignableFrom(value.ProblemType))
+          throw new ArgumentException("Algorithm is not single-objective!");
         if (algorithm == value) return;
-        if (algorithm != null) DeregisterAlgorithmEvents();
+        if (algorithm != null) {
+          DeregisterAlgorithmEvents();
+          RemoveAlgorithmAnalyzers();
+        }
         algorithm = value;
         if (algorithm != null) {
           RegisterAlgorithmEvents();
+          AddAlgorithmAnalyzers();
         }
         OnPropertyChanged("Algorithm");
@@ -228 +251 @@
     }
 
+    private ISingleObjectiveHeuristicOptimizationProblem problem;
+
     [StorableConstructor]
-    private IndepdentRandomRestartRun(bool deserializing) : base(deserializing) { }
-    private IndepdentRandomRestartRun(IndepdentRandomRestartRun original, Cloner cloner)
+    private IndepdentRandomRestarter(bool deserializing) : base(deserializing) { }
+    private IndepdentRandomRestarter(IndepdentRandomRestarter original, Cloner cloner)
       : base(original, cloner) {
       terminationCriterium = original.terminationCriterium;
       maximumExecutionTime = original.maximumExecutionTime;
       maximumEvaluations = original.maximumEvaluations;
+      moveCostPerSolution = original.moveCostPerSolution;
       targetValue = original.targetValue;
+      maximization = original.maximization;
       executionTime = original.executionTime;
       evaluations = original.evaluations;
@@ -243 +270 @@
       lastAlgorithmEvaluatedMoves = original.lastAlgorithmEvaluatedMoves;
 
+      perClockAnalyzer = cloner.Clone(original.perClockAnalyzer);
+      perEvaluationsAnalyzer = cloner.Clone(original.perEvaluationsAnalyzer);
+
       algorithm = cloner.Clone(original.algorithm);
       runs = cloner.Clone(original.runs);
 
+      ExecutionState = original.ExecutionState;
+
       Initialize();
     }
-    public IndepdentRandomRestartRun()
+    public IndepdentRandomRestarter()
       : base() {
       name = ItemName;
       description = ItemDescription;
-      terminationCriterium = TerminationCriterium.OnlyByEvaluations;
+      terminationCriterium = TerminationCriterium.ByTargetAndEvaluations;
       maximumExecutionTime = TimeSpan.FromMinutes(1);
       maximumEvaluations = 10000000; // 10 mio
+      moveCostPerSolution = 1;
       targetValue = 0;
+      maximization = false;
       executionTime = TimeSpan.Zero;
       evaluations = 0;
@@ -263 +297 @@
       lastAlgorithmEvaluatedMoves = 0;
 
+      perClockAnalyzer = new QualityPerClockAnalyzer();
+      perEvaluationsAnalyzer = new QualityPerEvaluationsAnalyzer();
+
       Runs = new RunCollection { OptimizerName = Name };
       Initialize();
     }
-    public IndepdentRandomRestartRun(string name)
+    public IndepdentRandomRestarter(string name)
       : base(name) {
       description = ItemDescription;
-      terminationCriterium = TerminationCriterium.OnlyByEvaluations;
+      terminationCriterium = TerminationCriterium.ByTargetAndEvaluations;
       maximumExecutionTime = TimeSpan.FromMinutes(1);
       maximumEvaluations = 10000000; // 10 mio
+      moveCostPerSolution = 1;
       targetValue = 0;
+      maximization = false;
       executionTime = TimeSpan.Zero;
       evaluations = 0;
@@ -280 +319 @@
       lastAlgorithmEvaluatedMoves = 0;
 
+      perClockAnalyzer = new QualityPerClockAnalyzer();
+      perEvaluationsAnalyzer = new QualityPerEvaluationsAnalyzer();
+
       Runs = new RunCollection { OptimizerName = Name };
       Initialize();
     }
-    public IndepdentRandomRestartRun(string name, string description)
+    public IndepdentRandomRestarter(string name, string description)
       : base(name, description) {
-      terminationCriterium = TerminationCriterium.OnlyByEvaluations;
+      terminationCriterium = TerminationCriterium.ByTargetAndEvaluations;
       maximumExecutionTime = TimeSpan.FromMinutes(1);
       maximumEvaluations = 10000000; // 10 mio
+      moveCostPerSolution = 1;
       targetValue = 0;
+      maximization = false;
       executionTime = TimeSpan.Zero;
       evaluations = 0;
@@ -296 +340 @@
       lastAlgorithmEvaluatedMoves = 0;
 
+      perClockAnalyzer = new QualityPerClockAnalyzer();
+      perEvaluationsAnalyzer = new QualityPerEvaluationsAnalyzer();
+
       Runs = new RunCollection { OptimizerName = Name };
       Initialize();
@@ -302 +349 @@
     public override IDeepCloneable Clone(Cloner cloner) {
       if (ExecutionState == ExecutionState.Started) throw new InvalidOperationException(string.Format("Clone not allowed in execution state \"{0}\".", ExecutionState));
-      return new IndepdentRandomRestartRun(this, cloner);
+      return new IndepdentRandomRestarter(this, cloner);
     }
 
@@ -314 +361 @@
     }
 
+    private void Reset() {
+      ExecutionTime = TimeSpan.Zero;
+      Evaluations = 0;
+      BestSoFar = double.NaN;
+      lastAlgorithmExecutionTime = TimeSpan.Zero;
+      lastAlgorithmEvaluatedSolutions = 0;
+      lastAlgorithmEvaluatedMoves = 0;
+
+      CurrentRun = null;
+    }
+
     public void Prepare() {
       Prepare(false);
     }
     public void Prepare(bool clearRuns) {
-      executionTime = TimeSpan.Zero;
-      evaluations = 0;
-      lastAlgorithmExecutionTime = TimeSpan.Zero;
-      lastAlgorithmEvaluatedSolutions = 0;
-
-      Algorithm.Prepare(clearRuns);
+      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused) && (ExecutionState != ExecutionState.Stopped))
+        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
+      Reset();
+
+      if (Algorithm != null) {
+        Algorithm.Prepare(clearRuns);
+        ExecutionState = ExecutionState.Prepared;
+        OnPrepared();
+      }
     }
     public void Start() {
+      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
+        throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));
+
       if (ExecutionState == ExecutionState.Prepared) {
-        currentRun = new Run(Algorithm) {
+        CurrentRun = new Run(Algorithm) {
           Name = Algorithm.Name + " IRRRun" + Runs.Count
         };
+        if (!CurrentRun.Results.ContainsKey(ExecutionTimeResultName))
+          CurrentRun.Results.Add(ExecutionTimeResultName, new TimeSpanValue(TimeSpan.Zero));
+        CurrentRun.Results.Add(perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName, new IntValue(0));
+        CurrentRun.Results.Add(perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName, new IntValue(0));
+        CurrentRun.Results.Add(BestQualityResultName, new DoubleValue(Maximization ? double.MinValue : double.MaxValue));
       }
       Algorithm.Start();
+      ExecutionState = ExecutionState.Started;
+      OnStarted();
     }
     public void Pause() {
+      if (ExecutionState != ExecutionState.Started)
+        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
       Algorithm.Pause();
-    }
+      ExecutionState = ExecutionState.Paused;
+      OnPaused();
+    }
+
+    private bool forceStop = false;
     public void Stop() {
+      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
+        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".", ExecutionState));
+      forceStop = true;
       Algorithm.Stop();
+    }
+
+    private void AddAlgorithmAnalyzers() {
+      if (!Algorithm.Parameters.ContainsKey("Analyzer")) return;
+      var analyzerParam = Algorithm.Parameters["Analyzer"] as IValueParameter<MultiAnalyzer>;
+      if (analyzerParam == null) return;
+      if (!analyzerParam.Value.Operators.Contains(perClockAnalyzer))
+        analyzerParam.Value.Operators.Add(perClockAnalyzer);
+      if (!analyzerParam.Value.Operators.Contains(perEvaluationsAnalyzer))
+        analyzerParam.Value.Operators.Add(perEvaluationsAnalyzer);
+    }
+
+    private void RemoveAlgorithmAnalyzers() {
+      if (!Algorithm.Parameters.ContainsKey("Analyzer")) return;
+      var analyzerParam = Algorithm.Parameters["Analyzer"] as IValueParameter<MultiAnalyzer>;
+      if (analyzerParam == null) return;
+      analyzerParam.Value.Operators.Remove(perClockAnalyzer);
+      analyzerParam.Value.Operators.Remove(perEvaluationsAnalyzer);
     }
 
@@ -400 +498 @@
       algorithm.Stopped += Algorithm_Stopped;
       algorithm.ProblemChanged += Algorithm_ProblemChanged;
+      Algorithm_ProblemChanged(algorithm, EventArgs.Empty);
     }
     private void DeregisterAlgorithmEvents() {
@@ -415 +514 @@
     }
     private void Algorithm_ExecutionTimeChanged(object sender, EventArgs e) {
-      ExecutionTime += Algorithm.ExecutionTime - lastAlgorithmExecutionTime;
-      lastAlgorithmExecutionTime = Algorithm.ExecutionTime;
-
-      UpdateAlgorithmResults();
-
-      if (IsFinished) {
+      if (Algorithm.ExecutionState != ExecutionState.Started) return;
+
+      if (ExecutionState == ExecutionState.Started)
+        UpdateAlgorithmResults();
+
+      if (IsFinished && ExecutionState != ExecutionState.Stopped) {
         Algorithm.Stop();
       }
@@ -427 +526 @@
 
     private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
-      OnExecutionStateChanged();
+      //OnExecutionStateChanged();
     }
     private void Algorithm_Paused(object sender, EventArgs e) {
-      ExecutionTime += Algorithm.ExecutionTime - lastAlgorithmExecutionTime;
-      lastAlgorithmExecutionTime = Algorithm.ExecutionTime;
-
       UpdateAlgorithmResults();
       OnPaused();
     }
     private void Algorithm_Prepared(object sender, EventArgs e) {
-      OnPrepared();
+      lastAlgorithmEvaluatedSolutions = 0;
+      lastAlgorithmEvaluatedMoves = 0;
+      lastAlgorithmExecutionTime = TimeSpan.Zero;
     }
     private void Algorithm_Started(object sender, EventArgs e) {
-      OnStarted();
+      //OnStarted();
     }
     private void Algorithm_Stopped(object sender, EventArgs e) {
-      ExecutionTime += Algorithm.ExecutionTime - lastAlgorithmExecutionTime;
-      lastAlgorithmExecutionTime = Algorithm.ExecutionTime;
-
       var bestQuality = UpdateAlgorithmResults();
 
+      var execTime = ((TimeSpanValue)currentRun.Results[ExecutionTimeResultName]).Value;
       foreach (var result in Algorithm.Results) {
-        if (result.Name == "QualityPerClock") {
+        if (result.Name == perClockAnalyzer.QualityPerClockParameter.ResultName) {
           if (!currentRun.Results.ContainsKey(result.Name))
             currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
           else {
             var dt = (IndexedDataTable<double>)currentRun.Results[result.Name];
-            var execTime = ((TimeSpanValue)currentRun.Results["ExecutionTime"]).Value.TotalSeconds;
             var best = dt.Rows.First().Values.Last().Item2;
             var resultDt = (IndexedDataTable<double>)result.Value;
             foreach (var tupl in resultDt.Rows.First().Values) {
               if (Maximization && tupl.Item2 > best || !Maximization && tupl.Item2 < best) {
-                dt.Rows.First().Values.Add(Tuple.Create(execTime + tupl.Item1, tupl.Item2));
+                dt.Rows.First().Values.Add(Tuple.Create(execTime.TotalSeconds + tupl.Item1, tupl.Item2));
                 best = tupl.Item2;
               }
             }
           }
-        } else if (result.Name == "QualityPerEvaluations") {
+        } else if (result.Name == perEvaluationsAnalyzer.QualityPerEvaluationsParameter.ResultName) {
           if (!currentRun.Results.ContainsKey(result.Name))
             currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
           else {
             var dt = (IndexedDataTable<double>)currentRun.Results[result.Name];
-            var evalSols = ((DoubleValue)currentRun.Results["EvaluatedSolutions"]).Value;
-            var evalMoves = ((DoubleValue)currentRun.Results["EvaluatedMoves"]).Value;
+            var evalSols = ((IntValue)currentRun.Results[perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName]).Value;
+            var evalMoves = ((IntValue)currentRun.Results[perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName]).Value;
             var best = dt.Rows.First().Values.Last().Item2;
             var resultDt = (IndexedDataTable<double>)result.Value;
@@ -480 +575 @@
             }
           }
-        } else if (result.Name == "EvaluatedSolutions") {
-          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
-        } else if (result.Name == "EvaluatedMoves") {
-          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
-        } else if (result.Name == "ExecutionTime") {
-          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
-        } else if (result.Name == "BestQuality") {
-          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
+        } else if (result.Name == perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName) {
+          var evalSols = ((IntValue)currentRun.Results[perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName]);
+          evalSols.Value += ((IntValue)result.Value).Value;
+        } else if (result.Name == perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName) {
+          var evalMoves = ((IntValue)currentRun.Results[perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName]);
+          evalMoves.Value += ((IntValue)result.Value).Value;
+        } else if (result.Name == perEvaluationsAnalyzer.BestQualityParameter.ActualName) {
+          var best = ((DoubleValue)currentRun.Results[BestQualityResultName]).Value;
+          if (Maximization && best < bestQuality || !Maximization && best > bestQuality)
+            currentRun.Results[BestQualityResultName] = new DoubleValue(bestQuality);
         } else if (result.Name.ToLower().EndsWith("solution") && BestSoFar == bestQuality) {
-          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
+          if (currentRun.Results.ContainsKey(result.Name))
+            currentRun.Results[result.Name] = (IItem)result.Value.Clone();
+          else currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
         }
       }
-      foreach (var result in Algorithm.Results) {
-        if (result.Name == "QualityPerClock") {
-
-        } else if (result.Name == "QualityPerEvaluations") {
-        } else if (result.Name == "EvaluatedSolutions") {
-          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
-        } else if (result.Name == "EvaluatedMoves") {
-          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
-        } else if (result.Name == "ExecutionTime") {
-          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
-        } else if (result.Name == "BestQuality") {
-          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
-        } else if (result.Name.ToLower().EndsWith("solution") && BestSoFar == bestQuality) {
-          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
-        }
-      }
-
-
-      if (IsFinished) {
-
-      }
-
-      // TODO
-      var cloner = new Cloner();
-      var algRun = cloner.Clone(Algorithm.Runs.Last());
-      Runs.Add(algRun);
-      Algorithm.Runs.Clear();
-      OnStopped();
+      currentRun.Results[ExecutionTimeResultName] = new TimeSpanValue(execTime + Algorithm.ExecutionTime);
+
+      if (!forceStop && !IsFinished) {
+        Algorithm.Prepare();
+        Algorithm.Start();
+      } else {
+        forceStop = false;
+        Runs.Add(currentRun);
+        currentRun = null;
+        Algorithm.Prepare(true);
+        ExecutionState = ExecutionState.Stopped;
+        OnStopped();
+      }
     }
 
     private double UpdateAlgorithmResults() {
+      ExecutionTime += Algorithm.ExecutionTime - lastAlgorithmExecutionTime;
+      lastAlgorithmExecutionTime = Algorithm.ExecutionTime;
+
       IResult evaluationsResult;
-      if (Algorithm.Results.TryGetValue("EvaluatedSolutions", out evaluationsResult)) {
+      if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName, out evaluationsResult)) {
         var evals = ((IntValue)evaluationsResult.Value).Value;
         Evaluations += evals - lastAlgorithmEvaluatedSolutions;
         lastAlgorithmEvaluatedSolutions = evals;
       }
-      if (Algorithm.Results.TryGetValue("EvaluatedMoves", out evaluationsResult)) {
+      if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName, out evaluationsResult)) {
         var evals = ((IntValue)evaluationsResult.Value).Value;
         Evaluations += moveCostPerSolution * (evals - lastAlgorithmEvaluatedMoves);
         lastAlgorithmEvaluatedMoves = evals;
       }
-      if (Algorithm.Results.TryGetValue("BestQuality", out evaluationsResult)) {
-        var bestQuality = ((DoubleValue)evaluationsResult).Value;
+      if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.BestQualityParameter.ActualName, out evaluationsResult)) {
+        var bestQuality = ((DoubleValue)evaluationsResult.Value).Value;
         if (double.IsNaN(BestSoFar)
             || Maximization && bestQuality > BestSoFar
@@ -546 +633 @@
 
     private void Algorithm_ProblemChanged(object sender, EventArgs e) {
-      var soProblem = Algorithm.Problem as ISingleObjectiveHeuristicOptimizationProblem;
-      if (soProblem == null) return;
-      var maxParam = soProblem.MaximizationParameter as IValueParameter<BoolValue>;
+      if (problem != null) DeregisterProblemEvents();
+
+      problem = Algorithm.Problem as ISingleObjectiveHeuristicOptimizationProblem;
+      if (problem == null) return;
+      RegisterProblemEvents();
+
+      AddAlgorithmAnalyzers();
+
+      var maxParam = problem.MaximizationParameter as IValueParameter<BoolValue>;
       if (maxParam != null)
         Maximization = maxParam.Value.Value;
-      var bkParam = soProblem.BestKnownQualityParameter as IValueParameter<DoubleValue>;
+      var bkParam = problem.BestKnownQualityParameter as IValueParameter<DoubleValue>;
       if (bkParam != null && bkParam.Value != null)
         TargetValue = bkParam.Value.Value;
+
+      Reset();
+    }
+
+    private void RegisterProblemEvents() {
+      problem.Reset += ProblemOnReset;
+      problem.OperatorsChanged += ProblemOnOperatorsChanged;
+    }
+
+    private void DeregisterProblemEvents() {
+      problem.Reset -= ProblemOnReset;
+      problem.OperatorsChanged -= ProblemOnOperatorsChanged;
+    }
+
+    private void ProblemOnReset(object sender, EventArgs eventArgs) {
+      AddAlgorithmAnalyzers();
+    }
+
+    private void ProblemOnOperatorsChanged(object sender, EventArgs eventArgs) {
+      AddAlgorithmAnalyzers();
     }
     #endregion