Changeset 12803

Timestamp:

07/24/15 15:31:54 (9 years ago)

Author:

abeham

Message:

#2431: worked on RLD analysis

• started implementation of IRRRun
• renamed view from ECDF to RLD
• reverted algorithms (execution time)
• changed per clock analyzer

Location:

branches/PerformanceComparison

Files:

• HeuristicLab.Analysis/3.3/Optimizers (added)
• HeuristicLab.Analysis/3.3/Optimizers/IRRRun.cs (copied) (copied from branches/PerformanceComparison/HeuristicLab.Optimization/3.3/MetaOptimizers/TimeLimitRun.cs) (12 diffs)
• HeuristicLab.Analysis/3.3/QualityAnalysis/QualityPerClockAnalyzer.cs (modified) (4 diffs)
• HeuristicLab.Optimization.Views/3.3/HeuristicLab.Optimization.Views-3.3.csproj (modified) (1 diff)
• HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.Designer.cs (moved) (moved from branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionECDFView.Designer.cs) (1 diff)
• HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs (moved) (moved from branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionECDFView.cs) (3 diffs)
• HeuristicLab.Optimization/3.3/Algorithms/BasicAlgorithm.cs (modified) (2 diffs)
• HeuristicLab.Optimization/3.3/Algorithms/EngineAlgorithm.cs (modified) (2 diffs)

branches/PerformanceComparison/HeuristicLab.Analysis/3.3/Optimizers/IRRRun.cs

@@ -25 +25 @@
 using System.Drawing;
 using System.Linq;
-using System.Threading.Tasks;
-using HeuristicLab.Collections;
 using HeuristicLab.Common;
 using HeuristicLab.Common.Resources;
 using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
-namespace HeuristicLab.Optimization {
+namespace HeuristicLab.Analysis {
+  public enum TerminationCriterium { OnlyByTime, OnlyByEvaluations, OnlyByTarget, ByTargetAndTime, ByTargetAndEvaluations, WhicheverHitsFirst, WhicheverHitsLast }
   /// <summary>
   /// A run in which an algorithm is executed for a certain maximum time only.
   /// </summary>
-  [Item("Timelimit Run", "A run in which an optimizer is executed a certain maximum time.")]
-  [Creatable(CreatableAttribute.Categories.TestingAndAnalysis, Priority = 115)]
+  [Item("Independent Random Restart Run", "A run in which an optimizer is repeated until either a certain target value is reached or a maximum budget is exceeded.")]
+  [Creatable(CreatableAttribute.Categories.TestingAndAnalysis, Priority = 117)]
   [StorableClass]
-  public sealed class TimeLimitRun : NamedItem, IOptimizer, IStorableContent, INotifyPropertyChanged {
+  public sealed class IndepdentRandomRestartRun : NamedItem, IOptimizer, IStorableContent, INotifyPropertyChanged {
     public string Filename { get; set; }
 
@@ -51 +52 @@
     #endregion
 
-    private bool pausedForSnapshot = false;
-    private bool pausedForTermination = false;
+    [Storable]
+    private TerminationCriterium terminationCriterium;
+    public TerminationCriterium TerminationCriterium {
+      get { return terminationCriterium; }
+      set {
+        if (terminationCriterium == value) return;
+        terminationCriterium = value;
+        OnPropertyChanged("TerminationCriterium");
+      }
+    }
 
     [Storable]
@@ -66 +75 @@
 
     [Storable]
-    private int snapshotTimesIndex;
-    [Storable]
-    private ObservableList<TimeSpan> snapshotTimes;
-    public ObservableList<TimeSpan> SnapshotTimes {
-      get { return snapshotTimes; }
-      set {
-        if (snapshotTimes == value) return;
-        snapshotTimes = value;
-        snapshotTimes.Sort();
-        FindNextSnapshotTimeIndex(ExecutionTime);
-        OnPropertyChanged("SnapshotTimes");
-      }
-    }
-
-    [Storable]
-    private bool storeAlgorithmInEachSnapshot;
-    [Storable]
-    public bool StoreAlgorithmInEachSnapshot {
-      get { return storeAlgorithmInEachSnapshot; }
-      set {
-        if (storeAlgorithmInEachSnapshot == value) return;
-        storeAlgorithmInEachSnapshot = value;
-        OnPropertyChanged("StoreAlgorithmInEachSnapshot");
-      }
-    }
-
-    [Storable]
-    private RunCollection snapshots;
-    public RunCollection Snapshots {
-      get { return snapshots; }
-      set {
-        if (snapshots == value) return;
-        snapshots = value;
-        OnPropertyChanged("Snapshots");
-      }
-    }
-
-    #region Inherited Properties
+    private int maximumEvaluations;
+    public int MaximumEvaluations {
+      get { return maximumEvaluations; }
+      set {
+        if (maximumEvaluations == value) return;
+        maximumEvaluations = value;
+        OnPropertyChanged("MaximumEvaluations");
+      }
+    }
+
+    [Storable]
+    private double targetValue;
+    public double TargetValue {
+      get { return targetValue; }
+      set {
+        if (targetValue == value) return;
+        targetValue = value;
+        OnPropertyChanged("TargetValue");
+      }
+    }
+
+    [Storable]
+    private bool maximization;
+    public bool Maximization {
+      get { return maximization; }
+      private set {
+        if (maximization == value) return;
+        maximization = value;
+        OnPropertyChanged("Maximization");
+      }
+    }
+
+    [Storable]
+    private double moveCostPerSolution;
+    public double MoveCostPerSolution {
+      get { return moveCostPerSolution; }
+      set {
+        if (moveCostPerSolution == value) return;
+        moveCostPerSolution = value;
+        OnPropertyChanged("MoveCostPerSolution");
+      }
+    }
+
+
     public ExecutionState ExecutionState {
       get { return (Algorithm != null) ? Algorithm.ExecutionState : ExecutionState.Stopped; }
     }
 
+    private TimeSpan lastAlgorithmExecutionTime;
+    [Storable]
+    private TimeSpan executionTime;
     public TimeSpan ExecutionTime {
-      get { return (Algorithm != null) ? Algorithm.ExecutionTime : TimeSpan.FromSeconds(0); }
+      get { return executionTime; }
+      set {
+        if (executionTime == value) return;
+        executionTime = value;
+        OnPropertyChanged("ExecutionTime");
+        OnExecutionTimeChanged();
+      }
+    }
+
+    private int lastAlgorithmEvaluatedSolutions;
+    private int lastAlgorithmEvaluatedMoves;
+    [Storable]
+    private double evaluations;
+    public double Evaluations {
+      get { return evaluations; }
+      set {
+        if (evaluations == value) return;
+        evaluations = value;
+        OnPropertyChanged("Evaluations");
+      }
+    }
+
+    [Storable]
+    private double bestSoFar;
+    public double BestSoFar {
+      get { return bestSoFar; }
+      set {
+        if (bestSoFar == value) return;
+        bestSoFar = value;
+        OnPropertyChanged("BestSoFar");
+      }
+    }
+
+    [Storable]
+    private IRun currentRun;
+    public IRun CurrentRun {
+      get { return currentRun; }
+      private set {
+        if (currentRun == value) return;
+        currentRun = value;
+        OnPropertyChanged("CurrentRun");
+      }
     }
 
@@ -148 +207 @@
       }
     }
-    #endregion
+
+    private bool IsFinished {
+      get {
+        var timeHit = ExecutionTime >= MaximumExecutionTime;
+        var evalHit = Evaluations >= MaximumEvaluations;
+        var targetHit = (Maximization && BestSoFar >= TargetValue || !Maximization && BestSoFar <= TargetValue);
+
+        return timeHit && evalHit && targetHit
+          || timeHit && (TerminationCriterium == TerminationCriterium.OnlyByTime
+                      || TerminationCriterium == TerminationCriterium.ByTargetAndTime
+                      || TerminationCriterium == TerminationCriterium.WhicheverHitsFirst)
+          || evalHit && (TerminationCriterium == TerminationCriterium.OnlyByEvaluations
+                      || TerminationCriterium == TerminationCriterium.ByTargetAndEvaluations
+                      || TerminationCriterium == TerminationCriterium.WhicheverHitsFirst)
+          || targetHit && (TerminationCriterium == TerminationCriterium.OnlyByTarget
+                        || TerminationCriterium == TerminationCriterium.WhicheverHitsFirst
+                        || TerminationCriterium == TerminationCriterium.ByTargetAndTime
+                        || TerminationCriterium == TerminationCriterium.ByTargetAndEvaluations);
+      }
+    }
 
     [StorableConstructor]
-    private TimeLimitRun(bool deserializing) : base(deserializing) { }
-    private TimeLimitRun(TimeLimitRun original, Cloner cloner)
+    private IndepdentRandomRestartRun(bool deserializing) : base(deserializing) { }
+    private IndepdentRandomRestartRun(IndepdentRandomRestartRun original, Cloner cloner)
       : base(original, cloner) {
+      terminationCriterium = original.terminationCriterium;
       maximumExecutionTime = original.maximumExecutionTime;
-      snapshotTimes = new ObservableList<TimeSpan>(original.snapshotTimes);
-      snapshotTimesIndex = original.snapshotTimesIndex;
-      snapshots = cloner.Clone(original.snapshots);
-      storeAlgorithmInEachSnapshot = original.storeAlgorithmInEachSnapshot;
+      maximumEvaluations = original.maximumEvaluations;
+      targetValue = original.targetValue;
+      executionTime = original.executionTime;
+      evaluations = original.evaluations;
+      bestSoFar = original.bestSoFar;
+      lastAlgorithmExecutionTime = original.lastAlgorithmExecutionTime;
+      lastAlgorithmEvaluatedSolutions = original.lastAlgorithmEvaluatedSolutions;
+      lastAlgorithmEvaluatedMoves = original.lastAlgorithmEvaluatedMoves;
+
       algorithm = cloner.Clone(original.algorithm);
       runs = cloner.Clone(original.runs);
@@ -164 +248 @@
       Initialize();
     }
-    public TimeLimitRun()
+    public IndepdentRandomRestartRun()
       : base() {
       name = ItemName;
       description = ItemDescription;
-      maximumExecutionTime = TimeSpan.FromMinutes(.5);
-      snapshotTimes = new ObservableList<TimeSpan>(new[] {
-          TimeSpan.FromSeconds(5),
-          TimeSpan.FromSeconds(10),
-          TimeSpan.FromSeconds(15) });
-      snapshotTimesIndex = 0;
-      snapshots = new RunCollection();
+      terminationCriterium = TerminationCriterium.OnlyByEvaluations;
+      maximumExecutionTime = TimeSpan.FromMinutes(1);
+      maximumEvaluations = 10000000; // 10 mio
+      targetValue = 0;
+      executionTime = TimeSpan.Zero;
+      evaluations = 0;
+      bestSoFar = double.NaN;
+      lastAlgorithmExecutionTime = TimeSpan.Zero;
+      lastAlgorithmEvaluatedSolutions = 0;
+      lastAlgorithmEvaluatedMoves = 0;
+
       Runs = new RunCollection { OptimizerName = Name };
       Initialize();
     }
-    public TimeLimitRun(string name)
+    public IndepdentRandomRestartRun(string name)
       : base(name) {
       description = ItemDescription;
-      maximumExecutionTime = TimeSpan.FromMinutes(.5);
-      snapshotTimes = new ObservableList<TimeSpan>(new[] {
-          TimeSpan.FromSeconds(5),
-          TimeSpan.FromSeconds(10),
-          TimeSpan.FromSeconds(15) });
-      snapshotTimesIndex = 0;
+      terminationCriterium = TerminationCriterium.OnlyByEvaluations;
+      maximumExecutionTime = TimeSpan.FromMinutes(1);
+      maximumEvaluations = 10000000; // 10 mio
+      targetValue = 0;
+      executionTime = TimeSpan.Zero;
+      evaluations = 0;
+      bestSoFar = double.NaN;
+      lastAlgorithmExecutionTime = TimeSpan.Zero;
+      lastAlgorithmEvaluatedSolutions = 0;
+      lastAlgorithmEvaluatedMoves = 0;
+
       Runs = new RunCollection { OptimizerName = Name };
       Initialize();
     }
-    public TimeLimitRun(string name, string description)
+    public IndepdentRandomRestartRun(string name, string description)
       : base(name, description) {
-      maximumExecutionTime = TimeSpan.FromMinutes(.5);
-      snapshotTimes = new ObservableList<TimeSpan>(new[] {
-          TimeSpan.FromSeconds(5),
-          TimeSpan.FromSeconds(10),
-          TimeSpan.FromSeconds(15) });
-      snapshotTimesIndex = 0;
+      terminationCriterium = TerminationCriterium.OnlyByEvaluations;
+      maximumExecutionTime = TimeSpan.FromMinutes(1);
+      maximumEvaluations = 10000000; // 10 mio
+      targetValue = 0;
+      executionTime = TimeSpan.Zero;
+      evaluations = 0;
+      bestSoFar = double.NaN;
+      lastAlgorithmExecutionTime = TimeSpan.Zero;
+      lastAlgorithmEvaluatedSolutions = 0;
+      lastAlgorithmEvaluatedMoves = 0;
+
       Runs = new RunCollection { OptimizerName = Name };
       Initialize();
@@ -204 +302 @@
     public override IDeepCloneable Clone(Cloner cloner) {
       if (ExecutionState == ExecutionState.Started) throw new InvalidOperationException(string.Format("Clone not allowed in execution state \"{0}\".", ExecutionState));
-      return new TimeLimitRun(this, cloner);
+      return new IndepdentRandomRestartRun(this, cloner);
     }
 
@@ -214 +312 @@
     private void Initialize() {
       if (algorithm != null) RegisterAlgorithmEvents();
-      snapshotTimes.ItemsAdded += snapshotTimes_Changed;
-      snapshotTimes.ItemsMoved += snapshotTimes_Changed;
-      snapshotTimes.ItemsRemoved += snapshotTimes_Changed;
-      snapshotTimes.ItemsReplaced += snapshotTimes_Changed;
-      snapshotTimes.CollectionReset += snapshotTimes_Changed;
-    }
-
-    private void snapshotTimes_Changed(object sender, CollectionItemsChangedEventArgs<IndexedItem<TimeSpan>> e) {
-      if (e.Items.Any()) snapshotTimes.Sort();
-      FindNextSnapshotTimeIndex(ExecutionTime);
-    }
-
-    public void Snapshot() {
-      if (Algorithm == null || Algorithm.ExecutionState != ExecutionState.Paused) throw new InvalidOperationException("Snapshot not allowed in execution states other than Paused");
-      Task.Factory.StartNew(MakeSnapshot);
     }
 
@@ -235 +318 @@
     }
     public void Prepare(bool clearRuns) {
+      executionTime = TimeSpan.Zero;
+      evaluations = 0;
+      lastAlgorithmExecutionTime = TimeSpan.Zero;
+      lastAlgorithmEvaluatedSolutions = 0;
+
       Algorithm.Prepare(clearRuns);
     }
     public void Start() {
+      if (ExecutionState == ExecutionState.Prepared) {
+        currentRun = new Run(Algorithm) {
+          Name = Algorithm.Name + " IRRRun" + Runs.Count
+        };
+      }
       Algorithm.Start();
     }
@@ -306 +399 @@
       algorithm.Started += Algorithm_Started;
       algorithm.Stopped += Algorithm_Stopped;
+      algorithm.ProblemChanged += Algorithm_ProblemChanged;
     }
     private void DeregisterAlgorithmEvents() {
@@ -315 +409 @@
       algorithm.Started -= Algorithm_Started;
       algorithm.Stopped -= Algorithm_Stopped;
+      algorithm.ProblemChanged -= Algorithm_ProblemChanged;
     }
     private void Algorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
@@ -320 +415 @@
     }
     private void Algorithm_ExecutionTimeChanged(object sender, EventArgs e) {
-      if (snapshotTimesIndex < SnapshotTimes.Count && ExecutionTime >= SnapshotTimes[snapshotTimesIndex]
-        && !pausedForSnapshot) {
-        pausedForSnapshot = true;
-        Algorithm.Pause();
-      }
-      if (ExecutionTime >= MaximumExecutionTime && !pausedForTermination) {
-        pausedForTermination = true;
-        if (!pausedForSnapshot) Algorithm.Pause();
+      ExecutionTime += Algorithm.ExecutionTime - lastAlgorithmExecutionTime;
+      lastAlgorithmExecutionTime = Algorithm.ExecutionTime;
+
+      UpdateAlgorithmResults();
+
+      if (IsFinished) {
+        Algorithm.Stop();
       }
       OnExecutionTimeChanged();
     }
+
     private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
       OnExecutionStateChanged();
     }
     private void Algorithm_Paused(object sender, EventArgs e) {
-      var action = pausedForTermination ? ExecutionState.Stopped : (pausedForSnapshot ? ExecutionState.Started : ExecutionState.Paused);
-      if (pausedForSnapshot || pausedForTermination) {
-        pausedForSnapshot = pausedForTermination = false;
-        MakeSnapshot();
-        FindNextSnapshotTimeIndex(ExecutionTime);
-      }
+      ExecutionTime += Algorithm.ExecutionTime - lastAlgorithmExecutionTime;
+      lastAlgorithmExecutionTime = Algorithm.ExecutionTime;
+
+      UpdateAlgorithmResults();
       OnPaused();
-      if (action == ExecutionState.Started) Algorithm.Start();
-      else if (action == ExecutionState.Stopped) Algorithm.Stop();
     }
     private void Algorithm_Prepared(object sender, EventArgs e) {
-      snapshotTimesIndex = 0;
-      snapshots.Clear();
       OnPrepared();
     }
@@ -354 +443 @@
     }
     private void Algorithm_Stopped(object sender, EventArgs e) {
+      ExecutionTime += Algorithm.ExecutionTime - lastAlgorithmExecutionTime;
+      lastAlgorithmExecutionTime = Algorithm.ExecutionTime;
+
+      var bestQuality = UpdateAlgorithmResults();
+
+      foreach (var result in Algorithm.Results) {
+        if (result.Name == "QualityPerClock") {
+          if (!currentRun.Results.ContainsKey(result.Name))
+            currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
+          else {
+            var dt = (IndexedDataTable<double>)currentRun.Results[result.Name];
+            var execTime = ((TimeSpanValue)currentRun.Results["ExecutionTime"]).Value.TotalSeconds;
+            var best = dt.Rows.First().Values.Last().Item2;
+            var resultDt = (IndexedDataTable<double>)result.Value;
+            foreach (var tupl in resultDt.Rows.First().Values) {
+              if (Maximization && tupl.Item2 > best || !Maximization && tupl.Item2 < best) {
+                dt.Rows.First().Values.Add(Tuple.Create(execTime + tupl.Item1, tupl.Item2));
+                best = tupl.Item2;
+              }
+            }
+          }
+        } else if (result.Name == "QualityPerEvaluations") {
+          if (!currentRun.Results.ContainsKey(result.Name))
+            currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
+          else {
+            var dt = (IndexedDataTable<double>)currentRun.Results[result.Name];
+            var evalSols = ((DoubleValue)currentRun.Results["EvaluatedSolutions"]).Value;
+            var evalMoves = ((DoubleValue)currentRun.Results["EvaluatedMoves"]).Value;
+            var best = dt.Rows.First().Values.Last().Item2;
+            var resultDt = (IndexedDataTable<double>)result.Value;
+            foreach (var tupl in resultDt.Rows.First().Values) {
+              if (Maximization && tupl.Item2 > best || !Maximization && tupl.Item2 < best) {
+                dt.Rows.First().Values.Add(Tuple.Create(evalSols + moveCostPerSolution * evalMoves + tupl.Item1, tupl.Item2));
+                best = tupl.Item2;
+              }
+            }
+          }
+        } else if (result.Name == "EvaluatedSolutions") {
+          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
+        } else if (result.Name == "EvaluatedMoves") {
+          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
+        } else if (result.Name == "ExecutionTime") {
+          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
+        } else if (result.Name == "BestQuality") {
+          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
+        } else if (result.Name.ToLower().EndsWith("solution") && BestSoFar == bestQuality) {
+          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
+        }
+      }
+      foreach (var result in Algorithm.Results) {
+        if (result.Name == "QualityPerClock") {
+
+        } else if (result.Name == "QualityPerEvaluations") {
+        } else if (result.Name == "EvaluatedSolutions") {
+          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
+        } else if (result.Name == "EvaluatedMoves") {
+          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
+        } else if (result.Name == "ExecutionTime") {
+          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
+        } else if (result.Name == "BestQuality") {
+          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
+        } else if (result.Name.ToLower().EndsWith("solution") && BestSoFar == bestQuality) {
+          currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
+        }
+      }
+
+
+      if (IsFinished) {
+
+      }
+
+      // TODO
       var cloner = new Cloner();
       var algRun = cloner.Clone(Algorithm.Runs.Last());
-      var clonedSnapshots = cloner.Clone(snapshots);
-      algRun.Results.Add("TimeLimitRunSnapshots", clonedSnapshots);
       Runs.Add(algRun);
       Algorithm.Runs.Clear();
       OnStopped();
     }
+
+    private double UpdateAlgorithmResults() {
+      IResult evaluationsResult;
+      if (Algorithm.Results.TryGetValue("EvaluatedSolutions", out evaluationsResult)) {
+        var evals = ((IntValue)evaluationsResult.Value).Value;
+        Evaluations += evals - lastAlgorithmEvaluatedSolutions;
+        lastAlgorithmEvaluatedSolutions = evals;
+      }
+      if (Algorithm.Results.TryGetValue("EvaluatedMoves", out evaluationsResult)) {
+        var evals = ((IntValue)evaluationsResult.Value).Value;
+        Evaluations += moveCostPerSolution * (evals - lastAlgorithmEvaluatedMoves);
+        lastAlgorithmEvaluatedMoves = evals;
+      }
+      if (Algorithm.Results.TryGetValue("BestQuality", out evaluationsResult)) {
+        var bestQuality = ((DoubleValue)evaluationsResult).Value;
+        if (double.IsNaN(BestSoFar)
+            || Maximization && bestQuality > BestSoFar
+            || !Maximization && bestQuality < BestSoFar)
+          BestSoFar = bestQuality;
+        return bestQuality;
+      }
+      return double.NaN;
+    }
+
+    private void Algorithm_ProblemChanged(object sender, EventArgs e) {
+      var soProblem = Algorithm.Problem as ISingleObjectiveHeuristicOptimizationProblem;
+      if (soProblem == null) return;
+      var maxParam = soProblem.MaximizationParameter as IValueParameter<BoolValue>;
+      if (maxParam != null)
+        Maximization = maxParam.Value.Value;
+      var bkParam = soProblem.BestKnownQualityParameter as IValueParameter<DoubleValue>;
+      if (bkParam != null && bkParam.Value != null)
+        TargetValue = bkParam.Value.Value;
+    }
     #endregion
     #endregion
-
-    private void FindNextSnapshotTimeIndex(TimeSpan reference) {
-      var index = 0;
-      while (index < snapshotTimes.Count && snapshotTimes[index] <= reference) {
-        index++;
-      };
-      snapshotTimesIndex = index;
-    }
-
-    private void MakeSnapshot() {
-      string time = Math.Round(ExecutionTime.TotalSeconds, 1).ToString("0.0");
-      string runName = "Snapshot " + time + "s " + algorithm.Name;
-      var changed = false;
-      if (StoreAlgorithmInEachSnapshot && !Algorithm.StoreAlgorithmInEachRun) {
-        Algorithm.StoreAlgorithmInEachRun = true;
-        changed = true;
-      } else if (!StoreAlgorithmInEachSnapshot && Algorithm.StoreAlgorithmInEachRun) {
-        Algorithm.StoreAlgorithmInEachRun = false;
-        changed = true;
-      }
-      var run = new Run(runName, Algorithm);
-      if (changed)
-        Algorithm.StoreAlgorithmInEachRun = !Algorithm.StoreAlgorithmInEachRun;
-
-      snapshots.Add(run);
-    }
   }
 }

branches/PerformanceComparison/HeuristicLab.Analysis/3.3/QualityAnalysis/QualityPerClockAnalyzer.cs

@@ -42 +42 @@
     }
 
-    public IResultParameter<TimeSpanValue> ExecutionTimeParameter {
-      get { return (IResultParameter<TimeSpanValue>)Parameters["Execution Time"]; }
+    public ILookupParameter<DateTimeValue> LastUpdateTimeParameter {
+      get { return (ILookupParameter<DateTimeValue>)Parameters["LastUpdateTime"]; }
     }
 
@@ -56 +56 @@
       : base() {
       Parameters.Add(new LookupParameter<DoubleValue>("BestQuality", "The quality value that should be compared."));
-      Parameters.Add(new ResultParameter<TimeSpanValue>("Execution Time", "The execution time."));
+      Parameters.Add(new LookupParameter<DateTimeValue>("LastUpdateTime", "The time the analyzer was last run."));
       Parameters.Add(new ResultParameter<IndexedDataTable<double>>("QualityPerClock", "Data table containing the first hitting graph with elapsed wall clock time (in seconds) as the x-axis."));
       QualityPerClockParameter.DefaultValue = new IndexedDataTable<double>("Quality per Clock") {
@@ -75 +75 @@
 
     public override IOperation Apply() {
-      var executionTime = Math.Max(ExecutionTimeParameter.ResultValue.Value.TotalSeconds, 0.2);
+      var lastUpdateTime = LastUpdateTimeParameter.ActualValue;
+      if (lastUpdateTime == null) {
+        lastUpdateTime = new DateTimeValue(DateTime.UtcNow.AddMilliseconds(-1));
+        LastUpdateTimeParameter.ActualValue = lastUpdateTime;
+      }
+      var now = DateTime.UtcNow;
+
       var bestQuality = BestQualityParameter.ActualValue.Value;
 
@@ -81 +87 @@
       var values = dataTable.Rows["First-hit Graph"].Values;
       if (values.Count == 0 || values.Last().Item2 != bestQuality)
-        dataTable.Rows["First-hit Graph"].Values.Add(Tuple.Create(executionTime, bestQuality));
+        dataTable.Rows["First-hit Graph"].Values.Add(Tuple.Create((now - lastUpdateTime.Value).TotalSeconds, bestQuality));
 
+      lastUpdateTime.Value = now;
       return base.Apply();
     }

branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/HeuristicLab.Optimization.Views-3.3.csproj

@@ -338 +338 @@
       <DependentUpon>RunCollectionEqualityConstraintView.cs</DependentUpon>
     </Compile>
-    <Compile Include="RunCollectionViews\RunCollectionECDFView.cs">
-      <SubType>UserControl</SubType>
-    </Compile>
-    <Compile Include="RunCollectionViews\RunCollectionECDFView.Designer.cs">
-      <DependentUpon>RunCollectionECDFView.cs</DependentUpon>
+    <Compile Include="RunCollectionViews\RunCollectionRLDView.cs">
+      <SubType>UserControl</SubType>
+    </Compile>
+    <Compile Include="RunCollectionViews\RunCollectionRLDView.Designer.cs">
+      <DependentUpon>RunCollectionRLDView.cs</DependentUpon>
     </Compile>
     <Compile Include="RunCollectionViews\RunCollectionTableView.cs">

branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.Designer.cs

@@ -20 +20 @@
 #endregion
 namespace HeuristicLab.Optimization.Views {
-  partial class RunCollectionECDFView {
+  partial class RunCollectionRLDView {
     /// <summary> 
     /// Required designer variable.

branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs

@@ -32 +32 @@
 
 namespace HeuristicLab.Optimization.Views {
-  [View("ECDF Performance Comparison")]
+  [View("Run-length Distribution View")]
   [Content(typeof(RunCollection), false)]
-  public partial class RunCollectionECDFView : ItemView {
+  public partial class RunCollectionRLDView : ItemView {
     private const string AllRuns = "All Runs";
 
@@ -48 +48 @@
     }
 
-    public RunCollectionECDFView() {
+    public RunCollectionRLDView() {
       InitializeComponent();
       combinedDataTable = new IndexedDataTable<double>("Combined DataTable", "A data table containing the ECDF of each of a number of groups.") {
@@ -253 +253 @@
 
     private void UpdateCaption() {
-      Caption = Content != null ? Content.OptimizerName + " ECDF Comparison" : ViewAttribute.GetViewName(GetType());
+      Caption = Content != null ? Content.OptimizerName + " RLD View" : ViewAttribute.GetViewName(GetType());
     }
 

branches/PerformanceComparison/HeuristicLab.Optimization/3.3/Algorithms/BasicAlgorithm.cs

@@ -24 +24 @@
 using System.Threading.Tasks;
 using HeuristicLab.Common;
-using HeuristicLab.Data;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
@@ -36 +35 @@
     public override ResultCollection Results {
       get { return results; }
-    }
-
-    protected override void OnExecutionTimeChanged() {
-      base.OnExecutionTimeChanged();
-      IResult result;
-      if (!Results.TryGetValue("Execution Time", out result))
-        Results.Add(new Result("Execution Time", new TimeSpanValue(ExecutionTime)));
-      else ((TimeSpanValue)result.Value).Value = ExecutionTime;
     }
 

branches/PerformanceComparison/HeuristicLab.Optimization/3.3/Algorithms/EngineAlgorithm.cs

@@ -22 +22 @@
 using System;
 using System.Linq;
-using System.Reflection;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
-using HeuristicLab.Data;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.PluginInfrastructure;
@@ -73 +71 @@
     }
 
-    protected override void OnExecutionTimeChanged() {
-      base.OnExecutionTimeChanged();
-      IResult result;
-      if (!Results.TryGetValue("Execution Time", out result))
-        Results.Add(new Result("Execution Time", new TimeSpanValue(ExecutionTime)));
-      else ((TimeSpanValue)result.Value).Value = ExecutionTime;
-    }
-
     public override ResultCollection Results {
       get {