Changeset 12771

Timestamp:

07/17/15 09:25:44 (9 years ago)

Author:

abeham

Message:

#2431:

• Added run collection view
• Changed name of analyzers
• Modified algorithms to include Execution Time as a result

Location:

branches/PerformanceComparison

Files:

• HeuristicLab.Analysis.Views/3.3/IndexedDataTableView.cs (modified) (2 diffs)
• HeuristicLab.Analysis/3.3/HeuristicLab.Analysis-3.3.csproj (modified) (1 diff)
• HeuristicLab.Analysis/3.3/QualityAnalysis/QualityPerClockAnalyzer.cs (moved) (moved from branches/PerformanceComparison/HeuristicLab.Analysis/3.3/QualityAnalysis/QualityVsEvaluationsAnalyzer.cs) (2 diffs)
• HeuristicLab.Analysis/3.3/QualityAnalysis/QualityPerEvaluationsAnalyzer.cs (copied) (copied from branches/PerformanceComparison/HeuristicLab.Analysis/3.3/QualityAnalysis/QualityVsEvaluationsAnalyzer.cs) (2 diffs)
• HeuristicLab.Optimization.Views/3.3/HeuristicLab.Optimization.Views-3.3.csproj (modified) (1 diff)
• HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionECDFView.Designer.cs (copied) (copied from branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionChartAggregationView.Designer.cs) (7 diffs)
• HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionECDFView.cs (copied) (copied from branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionChartAggregationView.cs) (10 diffs)
• HeuristicLab.Optimization/3.3/Algorithms/Algorithm.cs (modified) (1 diff)
• HeuristicLab.Optimization/3.3/Algorithms/BasicAlgorithm.cs (modified) (2 diffs)
• HeuristicLab.Optimization/3.3/Algorithms/EngineAlgorithm.cs (modified) (2 diffs)

branches/PerformanceComparison/HeuristicLab.Analysis.Views/3.3/IndexedDataTableView.cs

@@ -165 +165 @@
       series.BorderDashStyle = ChartDashStyle.Solid;
       series.BorderColor = Color.Empty;
-      if (!typeof(T).Equals(typeof(DateTime)))
-        series.IsXValueIndexed = true;
 
       if (row.VisualProperties.Color != Color.Empty)
@@ -236 +234 @@
         area.AxisX.IntervalAutoMode = IntervalAutoMode.VariableCount;
       }
+
+      area.AxisX.IsLogarithmic = Content.VisualProperties.XAxisLogScale;
+      area.AxisX2.IsLogarithmic = Content.VisualProperties.SecondXAxisLogScale;
+      area.AxisY.IsLogarithmic = Content.VisualProperties.YAxisLogScale;
+      area.AxisY2.IsLogarithmic = Content.VisualProperties.SecondYAxisLogScale;
     }
 

branches/PerformanceComparison/HeuristicLab.Analysis/3.3/HeuristicLab.Analysis-3.3.csproj

@@ -211 +211 @@
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="QualityAnalysis\QualityDistributionAnalyzer.cs" />
-    <Compile Include="QualityAnalysis\QualityVsEvaluationsAnalyzer.cs" />
+    <Compile Include="QualityAnalysis\QualityPerClockAnalyzer.cs" />
+    <Compile Include="QualityAnalysis\QualityPerEvaluationsAnalyzer.cs" />
     <Compile Include="QualityAnalysis\ScaledQualityDifferenceAnalyzer.cs" />
     <Compile Include="Statistics\BonferroniHolm.cs" />

branches/PerformanceComparison/HeuristicLab.Analysis/3.3/QualityAnalysis/QualityPerClockAnalyzer.cs

@@ -30 +30 @@
 
 namespace HeuristicLab.Analysis {
-  [Item("QualityVsEvaluationsAnalyzer", @"Creates a plot of the solution quality with respect to the number of evaluated solutions.")]
+  [Item("QualityPerClockAnalyzer", @"Creates a plot of the solution quality with respect to the elapsed wall-clock time.")]
   [StorableClass]
-  public class QualityVsEvaluationsAnalyzer : SingleSuccessorOperator, IAnalyzer, ISingleObjectiveOperator {
+  public class QualityPerClockAnalyzer : SingleSuccessorOperator, IAnalyzer, ISingleObjectiveOperator {
     public virtual bool EnabledByDefault {
       get { return false; }
@@ -40 +40 @@
       get { return (ILookupParameter<DoubleValue>)Parameters["BestQuality"]; }
     }
-    public ILookupParameter<IntValue> EvaluatedSolutionsParameter {
-      get { return (ILookupParameter<IntValue>)Parameters["EvaluatedSolutions"]; }
+
+    public IResultParameter<TimeSpanValue> ExecutionTimeParameter {
+      get { return (IResultParameter<TimeSpanValue>)Parameters["Execution Time"]; }
     }
-    public IResultParameter<IndexedDataTable<int>> QualityVsEvaluationsParameter {
-      get { return (IResultParameter<IndexedDataTable<int>>)Parameters["QualityVsEvaluations"]; }
+
+    public IResultParameter<IndexedDataTable<double>> QualityPerClockParameter {
+      get { return (IResultParameter<IndexedDataTable<double>>)Parameters["QualityPerClock"]; }
     }
 
     [StorableConstructor]
-    protected QualityVsEvaluationsAnalyzer(bool deserializing) : base(deserializing) { }
-    protected QualityVsEvaluationsAnalyzer(QualityVsEvaluationsAnalyzer original, Cloner cloner) : base(original, cloner) { }
-    public QualityVsEvaluationsAnalyzer()
+    protected QualityPerClockAnalyzer(bool deserializing) : base(deserializing) { }
+    protected QualityPerClockAnalyzer(QualityPerClockAnalyzer original, Cloner cloner) : base(original, cloner) { }
+    public QualityPerClockAnalyzer()
       : base() {
       Parameters.Add(new LookupParameter<DoubleValue>("BestQuality", "The quality value that should be compared."));
-      Parameters.Add(new LookupParameter<IntValue>("EvaluatedSolutions", "The quality value that should be compared."));
-      Parameters.Add(new ResultParameter<IndexedDataTable<int>>("QualityVsEvaluations", "Data table containing the first hitting graph with evaluations as the x-axis."));
-      QualityVsEvaluationsParameter.DefaultValue = new IndexedDataTable<int>("Quality vs Evaluations") {
-        Rows = { new IndexedDataRow<int>("First-hit Graph") { VisualProperties = { ChartType = DataRowVisualProperties.DataRowChartType.StepLine } } }
+      Parameters.Add(new ResultParameter<TimeSpanValue>("Execution Time", "The execution time."));
+      Parameters.Add(new ResultParameter<IndexedDataTable<double>>("QualityPerClock", "Data table containing the first hitting graph with elapsed wall clock time (in seconds) as the x-axis."));
+      QualityPerClockParameter.DefaultValue = new IndexedDataTable<double>("Quality per Clock") {
+        VisualProperties = {
+          XAxisTitle = "Elapsed time [s]",
+          YAxisTitle = "Quality"
+        },
+        Rows = { new IndexedDataRow<double>("First-hit Graph") { VisualProperties = {
+          ChartType = DataRowVisualProperties.DataRowChartType.StepLine,
+          LineWidth = 3
+        } } }
       };
     }
 
     public override IDeepCloneable Clone(Cloner cloner) {
-      return new QualityVsEvaluationsAnalyzer(this, cloner);
+      return new QualityPerClockAnalyzer(this, cloner);
     }
 
     public override IOperation Apply() {
+      var executionTime = Math.Max(ExecutionTimeParameter.ResultValue.Value.TotalSeconds, 0.001);
       var bestQuality = BestQualityParameter.ActualValue.Value;
-      var evaluations = EvaluatedSolutionsParameter.ActualValue.Value;
 
-      var dataTable = QualityVsEvaluationsParameter.ResultValue;
-      dataTable.Rows["First-hit Graph"].Values.Add(Tuple.Create(evaluations, bestQuality));
+      var dataTable = QualityPerClockParameter.ResultValue;
+      dataTable.Rows["First-hit Graph"].Values.Add(Tuple.Create(executionTime, bestQuality));
 
       return base.Apply();

branches/PerformanceComparison/HeuristicLab.Analysis/3.3/QualityAnalysis/QualityPerEvaluationsAnalyzer.cs

@@ -30 +30 @@
 
 namespace HeuristicLab.Analysis {
-  [Item("QualityVsEvaluationsAnalyzer", @"Creates a plot of the solution quality with respect to the number of evaluated solutions.")]
+  [Item("QualityPerEvaluationsAnalyzer", @"Creates a plot of the solution quality with respect to the number of evaluated solutions.")]
   [StorableClass]
-  public class QualityVsEvaluationsAnalyzer : SingleSuccessorOperator, IAnalyzer, ISingleObjectiveOperator {
+  public class QualityPerEvaluationsAnalyzer : SingleSuccessorOperator, IAnalyzer, ISingleObjectiveOperator {
     public virtual bool EnabledByDefault {
       get { return false; }
@@ -43 +43 @@
       get { return (ILookupParameter<IntValue>)Parameters["EvaluatedSolutions"]; }
     }
-    public IResultParameter<IndexedDataTable<int>> QualityVsEvaluationsParameter {
-      get { return (IResultParameter<IndexedDataTable<int>>)Parameters["QualityVsEvaluations"]; }
+    public IResultParameter<IndexedDataTable<double>> QualityPerEvaluationsParameter {
+      get { return (IResultParameter<IndexedDataTable<double>>)Parameters["QualityPerEvaluations"]; }
     }
 
     [StorableConstructor]
-    protected QualityVsEvaluationsAnalyzer(bool deserializing) : base(deserializing) { }
-    protected QualityVsEvaluationsAnalyzer(QualityVsEvaluationsAnalyzer original, Cloner cloner) : base(original, cloner) { }
-    public QualityVsEvaluationsAnalyzer()
+    protected QualityPerEvaluationsAnalyzer(bool deserializing) : base(deserializing) { }
+    protected QualityPerEvaluationsAnalyzer(QualityPerEvaluationsAnalyzer original, Cloner cloner) : base(original, cloner) { }
+    public QualityPerEvaluationsAnalyzer()
       : base() {
       Parameters.Add(new LookupParameter<DoubleValue>("BestQuality", "The quality value that should be compared."));
       Parameters.Add(new LookupParameter<IntValue>("EvaluatedSolutions", "The quality value that should be compared."));
-      Parameters.Add(new ResultParameter<IndexedDataTable<int>>("QualityVsEvaluations", "Data table containing the first hitting graph with evaluations as the x-axis."));
-      QualityVsEvaluationsParameter.DefaultValue = new IndexedDataTable<int>("Quality vs Evaluations") {
-        Rows = { new IndexedDataRow<int>("First-hit Graph") { VisualProperties = { ChartType = DataRowVisualProperties.DataRowChartType.StepLine } } }
+      Parameters.Add(new ResultParameter<IndexedDataTable<double>>("QualityPerEvaluations", "Data table containing the first hitting graph with evaluations as the x-axis."));
+      QualityPerEvaluationsParameter.DefaultValue = new IndexedDataTable<double>("Quality per Evaluations") {
+        VisualProperties = {
+          XAxisTitle = "Evaluations",
+          YAxisTitle = "Quality"
+        },
+        Rows = { new IndexedDataRow<double>("First-hit Graph") { VisualProperties = {
+          ChartType = DataRowVisualProperties.DataRowChartType.StepLine,
+          LineWidth = 3
+        } } }
       };
     }
 
     public override IDeepCloneable Clone(Cloner cloner) {
-      return new QualityVsEvaluationsAnalyzer(this, cloner);
+      return new QualityPerEvaluationsAnalyzer(this, cloner);
     }
 
     public override IOperation Apply() {
       var bestQuality = BestQualityParameter.ActualValue.Value;
-      var evaluations = EvaluatedSolutionsParameter.ActualValue.Value;
+      var evaluations = Math.Max(EvaluatedSolutionsParameter.ActualValue.Value, 1);
 
-      var dataTable = QualityVsEvaluationsParameter.ResultValue;
-      dataTable.Rows["First-hit Graph"].Values.Add(Tuple.Create(evaluations, bestQuality));
+      var dataTable = QualityPerEvaluationsParameter.ResultValue;
+      dataTable.Rows["First-hit Graph"].Values.Add(Tuple.Create((double)evaluations, bestQuality));
 
       return base.Apply();

branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/HeuristicLab.Optimization.Views-3.3.csproj

@@ -338 +338 @@
       <DependentUpon>RunCollectionEqualityConstraintView.cs</DependentUpon>
     </Compile>
+    <Compile Include="RunCollectionViews\RunCollectionECDFView.cs">
+      <SubType>UserControl</SubType>
+    </Compile>
+    <Compile Include="RunCollectionViews\RunCollectionECDFView.Designer.cs">
+      <DependentUpon>RunCollectionECDFView.cs</DependentUpon>
+    </Compile>
     <Compile Include="RunCollectionViews\RunCollectionTableView.cs">
       <SubType>UserControl</SubType>

branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionECDFView.Designer.cs

@@ -20 +20 @@
 #endregion
 namespace HeuristicLab.Optimization.Views {
-  partial class RunCollectionChartAggregationView {
+  partial class RunCollectionECDFView {
     /// <summary> 
     /// Required designer variable.
@@ -48 +48 @@
       this.viewHost = new HeuristicLab.MainForm.WindowsForms.ViewHost();
       this.label2 = new System.Windows.Forms.Label();
-      this.dataRowComboBox = new System.Windows.Forms.ComboBox();
+      this.groupComboBox = new System.Windows.Forms.ComboBox();
+      this.logScalingCheckBox = new System.Windows.Forms.CheckBox();
       this.SuspendLayout();
       // 
       // dataTableComboBox
       // 
-      this.dataTableComboBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left)
+      this.dataTableComboBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left) 
             | System.Windows.Forms.AnchorStyles.Right)));
       this.dataTableComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
       this.dataTableComboBox.FormattingEnabled = true;
-      this.dataTableComboBox.Location = new System.Drawing.Point(69, 3);
+      this.dataTableComboBox.Location = new System.Drawing.Point(69, 30);
       this.dataTableComboBox.Name = "dataTableComboBox";
       this.dataTableComboBox.Size = new System.Drawing.Size(455, 21);
@@ -66 +67 @@
       // 
       this.label1.AutoSize = true;
-      this.label1.Location = new System.Drawing.Point(3, 6);
+      this.label1.Location = new System.Drawing.Point(3, 33);
       this.label1.Name = "label1";
       this.label1.Size = new System.Drawing.Size(60, 13);
@@ -74 +75 @@
       // viewHost
       // 
-      this.viewHost.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom)
-            | System.Windows.Forms.AnchorStyles.Left)
+      this.viewHost.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) 
+            | System.Windows.Forms.AnchorStyles.Left) 
             | System.Windows.Forms.AnchorStyles.Right)));
       this.viewHost.Caption = "View";
@@ -83 +84 @@
       this.viewHost.Name = "viewHost";
       this.viewHost.ReadOnly = false;
-      this.viewHost.Size = new System.Drawing.Size(520, 314);
+      this.viewHost.Size = new System.Drawing.Size(520, 291);
       this.viewHost.TabIndex = 2;
       this.viewHost.ViewsLabelVisible = true;
@@ -91 +92 @@
       // 
       this.label2.AutoSize = true;
-      this.label2.Location = new System.Drawing.Point(3, 33);
+      this.label2.Location = new System.Drawing.Point(3, 6);
       this.label2.Name = "label2";
-      this.label2.Size = new System.Drawing.Size(55, 13);
+      this.label2.Size = new System.Drawing.Size(39, 13);
       this.label2.TabIndex = 3;
-      this.label2.Text = "DataRow:";
+      this.label2.Text = "Group:";
       // 
-      // dataRowComboBox
+      // groupComboBox
       // 
-      this.dataRowComboBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left)
+      this.groupComboBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left) 
             | System.Windows.Forms.AnchorStyles.Right)));
-      this.dataRowComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
-      this.dataRowComboBox.FormattingEnabled = true;
-      this.dataRowComboBox.Location = new System.Drawing.Point(69, 30);
-      this.dataRowComboBox.Name = "dataRowComboBox";
-      this.dataRowComboBox.Size = new System.Drawing.Size(455, 21);
-      this.dataRowComboBox.TabIndex = 4;
-      this.dataRowComboBox.SelectedIndexChanged += new System.EventHandler(this.dataRowComboBox_SelectedIndexChanged);
+      this.groupComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
+      this.groupComboBox.FormattingEnabled = true;
+      this.groupComboBox.Location = new System.Drawing.Point(69, 3);
+      this.groupComboBox.Name = "groupComboBox";
+      this.groupComboBox.Size = new System.Drawing.Size(455, 21);
+      this.groupComboBox.TabIndex = 4;
+      this.groupComboBox.SelectedIndexChanged += new System.EventHandler(this.groupComboBox_SelectedIndexChanged);
       // 
-      // RunCollectionChartAggregationView
+      // logScalingCheckBox
       // 
-      this.AutoScaleDimensions = new System.Drawing.SizeF(96F, 96F);
+      this.logScalingCheckBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Bottom | System.Windows.Forms.AnchorStyles.Left)));
+      this.logScalingCheckBox.AutoSize = true;
+      this.logScalingCheckBox.Location = new System.Drawing.Point(0, 354);
+      this.logScalingCheckBox.Name = "logScalingCheckBox";
+      this.logScalingCheckBox.Size = new System.Drawing.Size(112, 17);
+      this.logScalingCheckBox.TabIndex = 5;
+      this.logScalingCheckBox.Text = "logarithmic scaling";
+      this.logScalingCheckBox.UseVisualStyleBackColor = true;
+      this.logScalingCheckBox.CheckedChanged += new System.EventHandler(this.logScalingCheckBox_CheckedChanged);
+      // 
+      // RunCollectionECDFView
+      // 
       this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Inherit;
-      this.Controls.Add(this.dataRowComboBox);
+      this.Controls.Add(this.logScalingCheckBox);
+      this.Controls.Add(this.groupComboBox);
       this.Controls.Add(this.label2);
       this.Controls.Add(this.viewHost);
       this.Controls.Add(this.label1);
       this.Controls.Add(this.dataTableComboBox);
-      this.Name = "RunCollectionChartAggregationView";
+      this.Name = "RunCollectionECDFView";
       this.Size = new System.Drawing.Size(527, 374);
       this.ResumeLayout(false);
@@ -131 +144 @@
     private MainForm.WindowsForms.ViewHost viewHost;
     private System.Windows.Forms.Label label2;
-    private System.Windows.Forms.ComboBox dataRowComboBox;
+    private System.Windows.Forms.ComboBox groupComboBox;
+    private System.Windows.Forms.CheckBox logScalingCheckBox;
   }
 }

branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionECDFView.cs

@@ -24 +24 @@
 using System.ComponentModel;
 using System.Linq;
+using System.Windows.Forms;
 using HeuristicLab.Analysis;
 using HeuristicLab.Collections;
+using HeuristicLab.Common;
 using HeuristicLab.Core.Views;
 using HeuristicLab.MainForm;
 
 namespace HeuristicLab.Optimization.Views {
-  [View("Chart Aggregation")]
+  [View("ECDF Performance Comparison")]
   [Content(typeof(RunCollection), false)]
-  public partial class RunCollectionChartAggregationView : ItemView {
-    private const string AllDataRows = "All DataRows";
+  public partial class RunCollectionECDFView : ItemView {
+    private const string AllRuns = "All Runs";
 
     public new RunCollection Content {
@@ -40 +42 @@
     }
 
-    private int rowNumber;
     private bool suppressUpdates;
-    private readonly Dictionary<IRun, IEnumerable<DataRow>> runMapping;
-    private readonly DataTable combinedDataTable;
-    public DataTable CombinedDataTable {
+    private readonly IndexedDataTable<double> combinedDataTable;
+    public IndexedDataTable<double> CombinedDataTable {
       get { return combinedDataTable; }
     }
 
-    public RunCollectionChartAggregationView() {
+    public RunCollectionECDFView() {
       InitializeComponent();
-      runMapping = new Dictionary<IRun, IEnumerable<DataRow>>();
-      combinedDataTable = new DataTable("Combined DataTable", "A data table containing data rows from multiple runs.");
+      combinedDataTable = new IndexedDataTable<double>("Combined DataTable", "A data table containing the ECDF of each of a number of groups.") {
+        VisualProperties = { YAxisTitle = "Proportion of reached targets" }
+      };
       viewHost.Content = combinedDataTable;
       suppressUpdates = false;
@@ -80 +81 @@
         return;
       }
+      UpdateGroupComboBox();
       UpdateDataTableComboBox();
-      UpdateDataRowComboBox();
-      AddRuns(e.Items);
+      foreach (var run in e.Items)
+        RegisterRunEvents(run);
     }
     private void Content_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IRun> e) {
@@ -90 +92 @@
         return;
       }
+      UpdateGroupComboBox();
       UpdateDataTableComboBox();
-      UpdateDataRowComboBox();
-      RemoveRuns(e.Items);
+      foreach (var run in e.Items)
+        DeregisterRunEvents(run);
     }
     private void Content_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
@@ -100 +103 @@
         return;
       }
+      UpdateGroupComboBox();
       UpdateDataTableComboBox();
-      UpdateDataRowComboBox();
-      RemoveRuns(e.OldItems);
-      AddRuns(e.Items);
+      foreach (var run in e.OldItems)
+        DeregisterRunEvents(run);
     }
     private void Content_AlgorithmNameChanged(object sender, EventArgs e) {
@@ -118 +121 @@
       if (!suppressUpdates) {
         UpdateDataTableComboBox();
-        UpdateDataRowComboBox();
-        UpdateRuns(Content);
+        UpdateGroupComboBox();
+        UpdateRuns();
       }
     }
@@ -134 +137 @@
         Invoke((Action<object, PropertyChangedEventArgs>)run_PropertyChanged, sender, e);
       } else {
-        var run = (IRun)sender;
-        if (e.PropertyName == "Color" || e.PropertyName == "Visible")
-          UpdateRuns(new[] { run });
+        if (e.PropertyName == "Visible")
+          UpdateRuns();
       }
     }
@@ -144 +146 @@
       base.OnContentChanged();
       dataTableComboBox.Items.Clear();
-      dataRowComboBox.Items.Clear();
+      groupComboBox.Items.Clear();
       combinedDataTable.Rows.Clear();
-      runMapping.Clear();
 
       UpdateCaption();
       if (Content != null) {
+        UpdateGroupComboBox();
         UpdateDataTableComboBox();
       }
     }
 
-    private void RebuildCombinedDataTable() {
-      RemoveRuns(Content);
-      rowNumber = 0;
-      AddRuns(Content);
-    }
-
-    private void AddRuns(IEnumerable<IRun> runs) {
-      foreach (var run in runs) {
-        runMapping[run] = ExtractDataRowsFromRun(run).ToList();
-        RegisterRunEvents(run);
-      }
-      var dataRows = runs.Where(r => r.Visible && runMapping.ContainsKey(r)).SelectMany(r => runMapping[r]);
-      combinedDataTable.Rows.AddRange(dataRows);
-    }
-
-    private void RemoveRuns(IEnumerable<IRun> runs) {
-      var dataRows = runs.Where(r => runMapping.ContainsKey(r)).SelectMany(r => runMapping[r]).ToList();
-      foreach (var run in runs) {
-        if (!runMapping.ContainsKey(run)) continue;
-        runMapping.Remove(run);
-        DeregisterRunEvents(run);
-      }
-      combinedDataTable.Rows.RemoveRange(dataRows);
-    }
-
-    private void UpdateRuns(IEnumerable<IRun> runs) {
-      foreach (var run in runs) {
-        //update color
-        if (!runMapping.ContainsKey(run)) {
-          runMapping[run] = ExtractDataRowsFromRun(run).ToList();
-          RegisterRunEvents(run);
-        } else {
-          foreach (var dataRow in runMapping[run]) {
-            dataRow.VisualProperties.Color = run.Color;
+    private void UpdateRuns() {
+      if (InvokeRequired) {
+        Invoke((Action)UpdateRuns);
+        return;
+      }
+      SuspendRepaint();
+      try {
+        combinedDataTable.Rows.Clear();
+        var table = (string)dataTableComboBox.SelectedItem;
+        if (string.IsNullOrEmpty(table)) return;
+        var maximum = Content.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Max(y => y.Item2)).Max();
+        var minimum = Content.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Min(y => y.Item2)).Min();
+        var levels = Enumerable.Range(0, 51).Select(x => maximum - (x / 50.0) * (maximum - minimum)).ToArray();
+        var selectedGroup = (string)groupComboBox.SelectedItem;
+        if (string.IsNullOrEmpty(selectedGroup)) return;
+        List<Tuple<string, List<IRun>>> groupedRuns;
+        if (selectedGroup == AllRuns)
+          groupedRuns = new List<Tuple<string, List<IRun>>> { Tuple.Create(AllRuns, Content.ToList()) };
+        else groupedRuns = (from r in Content
+                            group r by r.Parameters[selectedGroup].ToString() into g
+                            select Tuple.Create(g.Key, g.ToList())).ToList();
+        var xAxisTitles = new HashSet<string>();
+        foreach (var group in groupedRuns) {
+          var hits = new SortedList<double, int>();
+          foreach (var run in group.Item2) {
+            var resultsTable = (IndexedDataTable<double>)run.Results[table];
+            xAxisTitles.Add(resultsTable.VisualProperties.XAxisTitle);
+            var graph = new LinkedList<Tuple<double, double>>(resultsTable.Rows.First().Values);
+            var current = graph.First.Next;
+            // prune convergence graph to obtain first hit times only
+            while (current != null) {
+              if (current.Value.Item2.IsAlmost(current.Previous.Value.Item2)) {
+                var h = current;
+                current = current.Previous;
+                graph.Remove(h);
+              }
+              current = current.Next;
+            }
+            var i = 0;
+            current = graph.First;
+            while (i < levels.Length && current != null) {
+              if (current.Value.Item2 < levels[i]) {
+                if (hits.ContainsKey(current.Value.Item1))
+                  hits[current.Value.Item1]++;
+                else hits[current.Value.Item1] = 1;
+                i++;
+              } else {
+                current = current.Next;
+              }
+            }
           }
+          var row = new IndexedDataRow<double>(group.Item1) { VisualProperties = { ChartType = DataRowVisualProperties.DataRowChartType.StepLine } };
+          var total = 0.0;
+          foreach (var h in hits) {
+            total += h.Value;
+            row.Values.Add(Tuple.Create(Math.Max(1, h.Key), total / (group.Item2.Count * levels.Length)));
+          }
+          combinedDataTable.Rows.Add(row);
         }
-      }
-      //update visibility - remove and add all rows to keep the same order as before
-      combinedDataTable.Rows.Clear();
-      combinedDataTable.Rows.AddRange(runMapping.Where(mapping => mapping.Key.Visible).SelectMany(mapping => mapping.Value));
-    }
-
-    private IEnumerable<DataRow> ExtractDataRowsFromRun(IRun run) {
-      var resultName = (string)dataTableComboBox.SelectedItem;
-      if (string.IsNullOrEmpty(resultName)) yield break;
-
-      var rowName = (string)dataRowComboBox.SelectedItem;
-      if (!run.Results.ContainsKey(resultName)) yield break;
-
-      var dataTable = (DataTable)run.Results[resultName];
-      foreach (var dataRow in dataTable.Rows) {
-        if (dataRow.Name != rowName && rowName != AllDataRows) continue;
-        rowNumber++;
-        var clonedRow = (DataRow)dataRow.Clone();
-        //row names must be unique -> add incremented number to the row name
-        clonedRow.Name = run.Name + "." + dataRow.Name + rowNumber;
-        clonedRow.VisualProperties.DisplayName = run.Name + "." + dataRow.Name;
-        clonedRow.VisualProperties.Color = run.Color;
-        yield return clonedRow;
+        combinedDataTable.VisualProperties.XAxisTitle = string.Join(" / ", xAxisTitles);
+      } finally { ResumeRepaint(true); }
+    }
+
+    private void UpdateGroupComboBox() {
+      string selectedItem = (string)groupComboBox.SelectedItem;
+
+      var groupings = Content.ParameterNames.ToArray();
+      groupComboBox.Items.Clear();
+      groupComboBox.Items.Add(AllRuns);
+      groupComboBox.Items.AddRange(groupings);
+      if (selectedItem != null && groupComboBox.Items.Contains(selectedItem)) {
+        groupComboBox.SelectedItem = selectedItem;
+      } else if (groupComboBox.Items.Count > 0) {
+        groupComboBox.SelectedItem = groupComboBox.Items[0];
       }
     }
@@ -222 +239 @@
       var dataTables = (from run in Content
                         from result in run.Results
-                        where result.Value is DataTable
+                        where result.Value is IndexedDataTable<double>
                         select result.Key).Distinct().ToArray();
 
@@ -234 +251 @@
 
     private void UpdateCaption() {
-      Caption = Content != null ? Content.OptimizerName + " Chart Aggregation" : ViewAttribute.GetViewName(GetType());
-    }
-
-    private void UpdateDataRowComboBox() {
-      string selectedItem = (string)dataRowComboBox.SelectedItem;
-
-      dataRowComboBox.Items.Clear();
-      var resultName = (string)dataTableComboBox.SelectedItem;
-      if (resultName == null) return;
-
-      var dataTables = from run in Content
-                       where run.Results.ContainsKey(resultName)
-                       select run.Results[resultName] as DataTable;
-      var rowNames = (from dataTable in dataTables
-                      from row in dataTable.Rows
-                      select row.Name).Distinct().ToArray();
-
-      dataRowComboBox.Items.AddRange(rowNames);
-      dataRowComboBox.Items.Add(AllDataRows);
-      if (selectedItem != null && dataRowComboBox.Items.Contains(selectedItem)) {
-        dataRowComboBox.SelectedItem = selectedItem;
-      } else if (dataRowComboBox.Items.Count > 0) {
-        dataRowComboBox.SelectedItem = dataRowComboBox.Items[0];
-      }
-    }
-
+      Caption = Content != null ? Content.OptimizerName + " ECDF Comparison" : ViewAttribute.GetViewName(GetType());
+    }
+
+    private void groupComboBox_SelectedIndexChanged(object sender, EventArgs e) {
+      UpdateRuns();
+    }
     private void dataTableComboBox_SelectedIndexChanged(object sender, EventArgs e) {
-      UpdateDataRowComboBox();
-    }
-    private void dataRowComboBox_SelectedIndexChanged(object sender, EventArgs e) {
-      if (suppressUpdates) return;
-      RebuildCombinedDataTable();
+      UpdateRuns();
+    }
+
+    private void logScalingCheckBox_CheckedChanged(object sender, EventArgs e) {
+      combinedDataTable.VisualProperties.XAxisLogScale = logScalingCheckBox.Checked;
     }
   }

branches/PerformanceComparison/HeuristicLab.Optimization/3.3/Algorithms/Algorithm.cs

@@ -226 +226 @@
     }
     public virtual void CollectResultValues(IDictionary<string, IItem> values) {
-      values.Add("Execution Time", new TimeSpanValue(ExecutionTime));
+      if (!Results.ContainsKey("Execution Time"))
+        values.Add("Execution Time", new TimeSpanValue(ExecutionTime));
       Results.CollectResultValues(values);
     }

branches/PerformanceComparison/HeuristicLab.Optimization/3.3/Algorithms/BasicAlgorithm.cs

@@ -24 +24 @@
 using System.Threading.Tasks;
 using HeuristicLab.Common;
+using HeuristicLab.Data;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
@@ -35 +36 @@
     public override ResultCollection Results {
       get { return results; }
+    }
+
+    protected override void OnExecutionTimeChanged() {
+      base.OnExecutionTimeChanged();
+      IResult result;
+      if (!Results.TryGetValue("Execution Time", out result))
+        Results.Add(new Result("Execution Time", new TimeSpanValue(ExecutionTime)));
+      else ((TimeSpanValue)result.Value).Value = ExecutionTime;
     }
 

branches/PerformanceComparison/HeuristicLab.Optimization/3.3/Algorithms/EngineAlgorithm.cs

@@ -22 +22 @@
 using System;
 using System.Linq;
+using System.Reflection;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Data;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.PluginInfrastructure;
@@ -71 +73 @@
     }
 
+    protected override void OnExecutionTimeChanged() {
+      base.OnExecutionTimeChanged();
+      IResult result;
+      if (!Results.TryGetValue("Execution Time", out result))
+        Results.Add(new Result("Execution Time", new TimeSpanValue(ExecutionTime)));
+      else ((TimeSpanValue)result.Value).Value = ExecutionTime;
+    }
+
     public override ResultCollection Results {
       get {