Changeset 12825 for branches/PerformanceComparison/HeuristicLab.Analysis/3.3/Optimizers

Timestamp:

08/02/15 01:14:55 (9 years ago)

Author:

abeham

Message:

#2431:

• removed output of RTs, RTus, ...
• added 2nd row to charts that state restarts until a target was achieved
• added option to include or exclude solution results

File:

• branches/PerformanceComparison/HeuristicLab.Analysis/3.3/Optimizers/IRRestarter.cs (modified) (12 diffs)

branches/PerformanceComparison/HeuristicLab.Analysis/3.3/Optimizers/IRRestarter.cs

@@ -44 +44 @@
     private const string BestQualityResultName = "BestQuality";
     private const string RandomRestartsResultName = "RandomRestarts";
-    private const string RTsResultName = "RTs";
-    private const string RTusResultName = "RTus";
-    private const string FEsResultName = "FEs";
-    private const string FEusResultName = "FEus";
+    private const string EvaluatedSolutionsResultName = "EvaluatedSolutions";
+    private const string EvaluatedMovesResultName = "EvaluatedMoves";
+    private const string QualityPerClockResultName = "QualityPerClock";
+    private const string QualityPerEvaluationsResultName = "QualityPerEvaluations";
 
     public string Filename { get; set; }
@@ -124 +124 @@
         perEvaluationsAnalyzer.MoveCostPerSolutionParameter.Value = new DoubleValue(moveCostPerSolution);
         OnPropertyChanged("MoveCostPerSolution");
+      }
+    }
+
+    [Storable]
+    private bool storeSolutionInRun;
+    public bool StoreSolutionInRun {
+      get { return storeSolutionInRun; }
+      set {
+        if (storeSolutionInRun == value) return;
+        storeSolutionInRun = value;
+        OnPropertyChanged("StoreSolutionInRun");
       }
     }
@@ -196 +207 @@
       get { return algorithm; }
       set {
-        if (value != null && !value.ProblemType.IsAssignableFrom(typeof(ISingleObjectiveHeuristicOptimizationProblem)))
-          throw new ArgumentException("Algorithm is not single-objective!");
         if (algorithm == value) return;
         if (algorithm != null) {
@@ -266 +275 @@
       maximumEvaluations = original.maximumEvaluations;
       moveCostPerSolution = original.moveCostPerSolution;
+      storeSolutionInRun = original.storeSolutionInRun;
       targetValue = original.targetValue;
       maximization = original.maximization;
@@ -291 +301 @@
       maximumEvaluations = 10000000; // 10 mio
       moveCostPerSolution = 1;
+      storeSolutionInRun = false;
       targetValue = 0;
       maximization = false;
@@ -311 +322 @@
       maximumEvaluations = 10000000; // 10 mio
       moveCostPerSolution = 1;
+      storeSolutionInRun = false;
       targetValue = 0;
       maximization = false;
@@ -330 +342 @@
       maximumEvaluations = 10000000; // 10 mio
       moveCostPerSolution = 1;
+      storeSolutionInRun = false;
       targetValue = 0;
       maximization = false;
@@ -391 +404 @@
         if (!CurrentRun.Results.ContainsKey(ExecutionTimeResultName))
           CurrentRun.Results.Add(ExecutionTimeResultName, new TimeSpanValue(TimeSpan.Zero));
-        CurrentRun.Results.Add(perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName, new IntValue(0));
-        CurrentRun.Results.Add(perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName, new IntValue(0));
+        // use double instead of int, otherwise one might run into int.MaxValue (at least with moves)
+        CurrentRun.Results.Add(EvaluatedSolutionsResultName, new DoubleValue(0));
+        CurrentRun.Results.Add(EvaluatedMovesResultName, new DoubleValue(0));
         CurrentRun.Results.Add(BestQualityResultName, new DoubleValue(Maximization ? double.MinValue : double.MaxValue));
         CurrentRun.Results.Add(RandomRestartsResultName, new IntValue(0));
-        CurrentRun.Results.Add(RTsResultName, new DoubleValue(0));
-        CurrentRun.Results.Add(RTusResultName, new DoubleValue(0));
-        CurrentRun.Results.Add(FEsResultName, new DoubleValue(0));
-        CurrentRun.Results.Add(FEusResultName, new DoubleValue(0));
-      }
+      }
+      if (Algorithm.ExecutionState == ExecutionState.Stopped)
+        Algorithm.Prepare(true);
       Algorithm.Start();
       ExecutionState = ExecutionState.Started;
@@ -417 +429 @@
         throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".", ExecutionState));
       forceStop = true;
-      Algorithm.Stop();
+      try {
+        Algorithm.Stop();
+      } catch (InvalidOperationException) {
+        // sometimes we hit the algorithm in an invalid state
+      }
     }
 
@@ -522 +538 @@
     }
     private void Algorithm_Stopped(object sender, EventArgs e) {
-      var bestQuality = UpdateAlgorithmResults();
+      ExecutionTime += Algorithm.ExecutionTime - lastAlgorithmExecutionTime;
+      lastAlgorithmExecutionTime = Algorithm.ExecutionTime;
 
       var execTime = ((TimeSpanValue)CurrentRun.Results[ExecutionTimeResultName]).Value;
-      double evaluationsInThisRun = 0;
-      foreach (var result in Algorithm.Results) {
-        if (result.Name == perClockAnalyzer.QualityPerClockParameter.ResultName) {
-          if (!CurrentRun.Results.ContainsKey(result.Name))
-            CurrentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
-          else {
-            var dt = (IndexedDataTable<double>)CurrentRun.Results[result.Name];
-            var best = dt.Rows.First().Values.Last().Item2;
-            var resultDt = (IndexedDataTable<double>)result.Value;
-            foreach (var tupl in resultDt.Rows.First().Values) {
-              if (Maximization && tupl.Item2 > best || !Maximization && tupl.Item2 < best) {
-                dt.Rows.First().Values.Add(Tuple.Create(execTime.TotalSeconds + tupl.Item1, tupl.Item2));
-                best = tupl.Item2;
-              }
-            }
+      var solEvals = ((DoubleValue)CurrentRun.Results[EvaluatedSolutionsResultName]).Value;
+      var movEvals = ((DoubleValue)CurrentRun.Results[EvaluatedMovesResultName]).Value;
+      var restarts = ((IntValue)CurrentRun.Results[RandomRestartsResultName]).Value;
+      var improvement = false;
+
+      IResult result;
+      if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName, out result)) {
+        var evals = ((IntValue)result.Value).Value;
+        Evaluations += evals;
+        CurrentRun.Results[EvaluatedSolutionsResultName] = new DoubleValue(solEvals + evals);
+      }
+      if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName, out result)) {
+        var evals = ((IntValue)result.Value).Value;
+        Evaluations += moveCostPerSolution * evals;
+        CurrentRun.Results[EvaluatedMovesResultName] = new DoubleValue(movEvals + evals);
+      }
+      if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.BestQualityParameter.ActualName, out result)) {
+        var bestQuality = ((DoubleValue)result.Value).Value;
+        if (double.IsNaN(BestSoFar)
+            || Maximization && bestQuality > BestSoFar
+            || !Maximization && bestQuality < BestSoFar) {
+          BestSoFar = bestQuality;
+          CurrentRun.Results[BestQualityResultName] = new DoubleValue(bestQuality);
+          improvement = true;
+        }
+      }
+      if (Algorithm.Results.TryGetValue(perClockAnalyzer.QualityPerClockParameter.ResultName, out result)) {
+        UpdateQualityPerClockResult((IndexedDataTable<double>)result.Value, execTime, restarts);
+      }
+      if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.QualityPerEvaluationsParameter.ResultName, out result)) {
+        UpdateQualityPerEvaluationsResult((IndexedDataTable<double>)result.Value, solEvals, movEvals, restarts);
+      }
+      if (StoreSolutionInRun) {
+        foreach (var r in Algorithm.Results) {
+          if (r.Name.ToLower().EndsWith("solution") && improvement) {
+            CurrentRun.Results[r.Name] = (IItem)r.Value.Clone();
           }
-        } else if (result.Name == perEvaluationsAnalyzer.QualityPerEvaluationsParameter.ResultName) {
-          if (!CurrentRun.Results.ContainsKey(result.Name))
-            CurrentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
-          else {
-            var dt = (IndexedDataTable<double>)CurrentRun.Results[result.Name];
-            var evalSols = ((IntValue)CurrentRun.Results[perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName]).Value;
-            var evalMoves = ((IntValue)CurrentRun.Results[perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName]).Value;
-            var best = dt.Rows.First().Values.Last().Item2;
-            var resultDt = (IndexedDataTable<double>)result.Value;
-            foreach (var tupl in resultDt.Rows.First().Values) {
-              if (Maximization && tupl.Item2 > best || !Maximization && tupl.Item2 < best) {
-                dt.Rows.First().Values.Add(Tuple.Create(evalSols + moveCostPerSolution * evalMoves + tupl.Item1, tupl.Item2));
-                best = tupl.Item2;
-              }
-            }
-          }
-        } else if (result.Name == perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName) {
-          var oldEvals = ((IntValue)CurrentRun.Results[perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName]).Value;
-          var newEvals = ((IntValue)result.Value).Value;
-          CurrentRun.Results[perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName] = new IntValue(oldEvals + newEvals);
-          evaluationsInThisRun += newEvals;
-        } else if (result.Name == perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName) {
-          var oldEvals = ((IntValue)CurrentRun.Results[perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName]).Value;
-          var newEvals = ((IntValue)result.Value).Value;
-          CurrentRun.Results[perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName] = new IntValue(oldEvals + newEvals);
-          evaluationsInThisRun += newEvals * moveCostPerSolution;
-        } else if (result.Name == perEvaluationsAnalyzer.BestQualityParameter.ActualName) {
-          var best = ((DoubleValue)CurrentRun.Results[BestQualityResultName]).Value;
-          if (Maximization && best < bestQuality || !Maximization && best > bestQuality)
-            CurrentRun.Results[BestQualityResultName] = new DoubleValue(bestQuality);
-        } else if (result.Name.ToLower().EndsWith("solution") && BestSoFar == bestQuality) {
-          if (CurrentRun.Results.ContainsKey(result.Name))
-            CurrentRun.Results[result.Name] = (IItem)result.Value.Clone();
-          else CurrentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
         }
       }
+
       CurrentRun.Results[ExecutionTimeResultName] = new TimeSpanValue(execTime + Algorithm.ExecutionTime);
 
@@ -584 +589 @@
 
       if (!forceStop && !IsFinished) {
-        var restarts = ((IntValue)CurrentRun.Results[RandomRestartsResultName]).Value;
-        if (restarts == 0) {
-          CurrentRun.Results[RTusResultName] = new DoubleValue(Algorithm.ExecutionTime.TotalSeconds);
-          CurrentRun.Results[FEusResultName] = new DoubleValue(evaluationsInThisRun);
-        } else {
-          var rtus = ((DoubleValue)CurrentRun.Results[RTusResultName]).Value;
-          var feus = ((DoubleValue)CurrentRun.Results[FEusResultName]).Value;
-          CurrentRun.Results[RTusResultName] = new DoubleValue(rtus * restarts / (restarts + 1.0) + Algorithm.ExecutionTime.TotalSeconds / (restarts + 1.0));
-          CurrentRun.Results[FEusResultName] = new DoubleValue(feus * restarts / (restarts + 1.0) + evaluationsInThisRun / (restarts + 1.0));
-        }
         CurrentRun.Results[RandomRestartsResultName] = new IntValue(restarts + 1);
         Algorithm.Prepare(true);
         Algorithm.Start();
       } else {
-        if (Maximization && BestSoFar >= TargetValue || !Maximization && BestSoFar <= TargetValue) {
-          CurrentRun.Results[RTsResultName] = (IItem)Algorithm.Results[perClockAnalyzer.QualityPerClockParameter.ResultName].Value.Clone();
-          CurrentRun.Results[FEsResultName] = (IItem)Algorithm.Results[perEvaluationsAnalyzer.QualityPerEvaluationsParameter.ResultName].Value.Clone();
-        }
         forceStop = false;
         Runs.Add(CurrentRun);
@@ -610 +601 @@
     }
 
-    private double UpdateAlgorithmResults() {
-      ExecutionTime += Algorithm.ExecutionTime - lastAlgorithmExecutionTime;
-      lastAlgorithmExecutionTime = Algorithm.ExecutionTime;
-
-      IResult evaluationsResult;
-      if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName, out evaluationsResult)) {
-        var evals = ((IntValue)evaluationsResult.Value).Value;
-        Evaluations += evals;
-      }
-      if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName, out evaluationsResult)) {
-        var evals = ((IntValue)evaluationsResult.Value).Value;
-        Evaluations += moveCostPerSolution * evals;
-      }
-      if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.BestQualityParameter.ActualName, out evaluationsResult)) {
-        var bestQuality = ((DoubleValue)evaluationsResult.Value).Value;
-        if (double.IsNaN(BestSoFar)
-            || Maximization && bestQuality > BestSoFar
-            || !Maximization && bestQuality < BestSoFar)
-          BestSoFar = bestQuality;
-        return bestQuality;
-      }
-      return double.NaN;
+    private void UpdateQualityPerClockResult(IndexedDataTable<double> perClock, TimeSpan execTime, int restarts) {
+      IndexedDataTable<double> dt;
+      if (!CurrentRun.Results.ContainsKey(QualityPerClockResultName)) {
+        dt = (IndexedDataTable<double>)perClock.Clone();
+        if (!dt.Rows.ContainsKey("Restarts"))
+          dt.Rows.Add(new IndexedDataRow<double>("Restarts") {
+            VisualProperties = {
+              ChartType = DataRowVisualProperties.DataRowChartType.StepLine,
+              SecondYAxis = true
+            }
+          });
+        foreach (var v in dt.Rows.First().Values)
+          dt.Rows["Restarts"].Values.Add(Tuple.Create(v.Item1, 0.0));
+        CurrentRun.Results.Add(QualityPerClockResultName, dt);
+      } else {
+        dt = (IndexedDataTable<double>)CurrentRun.Results[QualityPerClockResultName];
+        var best = dt.Rows.First().Values.Last().Item2;
+        foreach (var tupl in perClock.Rows.First().Values) {
+          if (Maximization && tupl.Item2 > best || !Maximization && tupl.Item2 < best) {
+            dt.Rows.First().Values.Add(Tuple.Create(execTime.TotalSeconds + tupl.Item1, tupl.Item2));
+            dt.Rows["Restarts"].Values.Add(Tuple.Create(execTime.TotalSeconds + tupl.Item1, (double)restarts));
+            best = tupl.Item2;
+          }
+        }
+      }
+      if (IsFinished) {
+        dt.Rows.First().Values.Add(Tuple.Create(ExecutionTime.TotalSeconds, BestSoFar));
+        dt.Rows["Restarts"].Values.Add(Tuple.Create(ExecutionTime.TotalSeconds, (double)restarts));
+      }
+    }
+
+    private void UpdateQualityPerEvaluationsResult(IndexedDataTable<double> perEvaluations, double solEvals, double movEvals, int restarts) {
+      IndexedDataTable<double> dt;
+      if (!CurrentRun.Results.ContainsKey(QualityPerEvaluationsResultName)) {
+        dt = (IndexedDataTable<double>)perEvaluations.Clone();
+        if (!dt.Rows.ContainsKey("Restarts"))
+          dt.Rows.Add(new IndexedDataRow<double>("Restarts") {
+            VisualProperties = {
+              ChartType = DataRowVisualProperties.DataRowChartType.StepLine,
+              SecondYAxis = true
+            }
+          });
+        foreach (var v in dt.Rows.First().Values)
+          dt.Rows["Restarts"].Values.Add(Tuple.Create(v.Item1, 0.0));
+        CurrentRun.Results.Add(QualityPerEvaluationsResultName, dt);
+      } else {
+        dt = (IndexedDataTable<double>)CurrentRun.Results[QualityPerEvaluationsResultName];
+        var best = dt.Rows.First().Values.Last().Item2;
+        foreach (var tupl in perEvaluations.Rows.First().Values) {
+          if (Maximization && tupl.Item2 > best || !Maximization && tupl.Item2 < best) {
+            dt.Rows.First().Values.Add(Tuple.Create(solEvals + moveCostPerSolution * movEvals + tupl.Item1, tupl.Item2));
+            dt.Rows["Restarts"].Values.Add(Tuple.Create(solEvals + moveCostPerSolution * movEvals + tupl.Item1, (double)restarts));
+            best = tupl.Item2;
+          }
+        }
+      }
+      if (IsFinished) {
+        dt.Rows.First().Values.Add(Tuple.Create(Evaluations, BestSoFar));
+        dt.Rows["Restarts"].Values.Add(Tuple.Create(Evaluations, (double)restarts));
+      }
     }