Changeset 14665

Timestamp:

02/11/17 01:05:41 (8 years ago)

Author:

abeham

Message:

#2457: updated branch to trunk

Location:

branches/PerformanceComparison

Files:

• HeuristicLab.Analysis (modified) (1 prop)
• HeuristicLab.Analysis.Views (modified) (1 prop)
• HeuristicLab.Optimization (modified) (1 prop)
• HeuristicLab.Optimization.Views (modified) (1 prop)
• HeuristicLab.Optimization.Views/3.3/HeuristicLab.Optimization.Views-3.3.csproj (modified) (1 diff)
• HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionParameterAnalysisView.Designer.cs (copied) (copied from trunk/sources/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionParameterAnalysisView.Designer.cs)
• HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionParameterAnalysisView.cs (copied) (copied from trunk/sources/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionParameterAnalysisView.cs)
• HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.Designer.cs (modified) (11 diffs)
• HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs (modified) (37 diffs)

branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/HeuristicLab.Optimization.Views-3.3.csproj

@@ -349 +349 @@
       <DependentUpon>RunCollectionEqualityConstraintView.cs</DependentUpon>
     </Compile>
+    <Compile Include="RunCollectionViews\RunCollectionParameterAnalysisView.cs">
+      <SubType>UserControl</SubType>
+    </Compile>
+    <Compile Include="RunCollectionViews\RunCollectionParameterAnalysisView.Designer.cs">
+      <DependentUpon>RunCollectionParameterAnalysisView.cs</DependentUpon>
+    </Compile>
     <Compile Include="RunCollectionViews\RunCollectionRLDView.cs">
       <SubType>UserControl</SubType>

branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.Designer.cs

@@ -64 +64 @@
       this.tabControl = new System.Windows.Forms.TabControl();
       this.byTargetTabPage = new System.Windows.Forms.TabPage();
+      this.relativeOrAbsoluteComboBox = new System.Windows.Forms.ComboBox();
       this.targetChart = new HeuristicLab.Visualization.ChartControlsExtensions.EnhancedChart();
+      this.markerCheckBox = new System.Windows.Forms.CheckBox();
       this.boundShadingCheckBox = new System.Windows.Forms.CheckBox();
       this.byCostTabPage = new System.Windows.Forms.TabPage();
       this.byCostViewHost = new HeuristicLab.MainForm.WindowsForms.ViewHost();
       this.budgetLogScalingCheckBox = new System.Windows.Forms.CheckBox();
-      this.eachOrAllBudgetsCheckBox = new System.Windows.Forms.CheckBox();
+      this.aggregateBudgetsCheckBox = new System.Windows.Forms.CheckBox();
       this.generateBudgetsButton = new System.Windows.Forms.Button();
       this.byTableTabPage = new System.Windows.Forms.TabPage();
@@ -143 +145 @@
       this.targetsTextBox.Location = new System.Drawing.Point(59, 8);
       this.targetsTextBox.Name = "targetsTextBox";
-      this.targetsTextBox.Size = new System.Drawing.Size(287, 20);
+      this.targetsTextBox.Size = new System.Drawing.Size(204, 20);
       this.targetsTextBox.TabIndex = 1;
       this.toolTip.SetToolTip(this.targetsTextBox, "The order of the targets is important, first to-hit targets\r\nshould be given firs" +
@@ -200 +202 @@
       this.budgetsTextBox.Location = new System.Drawing.Point(59, 8);
       this.budgetsTextBox.Name = "budgetsTextBox";
-      this.budgetsTextBox.Size = new System.Drawing.Size(326, 20);
+      this.budgetsTextBox.Size = new System.Drawing.Size(309, 20);
       this.budgetsTextBox.TabIndex = 6;
       this.budgetsTextBox.Validating += new System.ComponentModel.CancelEventHandler(this.budgetsTextBox_Validating);
@@ -221 +223 @@
       this.aggregateTargetsCheckBox.Checked = true;
       this.aggregateTargetsCheckBox.CheckState = System.Windows.Forms.CheckState.Checked;
-      this.aggregateTargetsCheckBox.Location = new System.Drawing.Point(352, 10);
+      this.aggregateTargetsCheckBox.Location = new System.Drawing.Point(374, 10);
       this.aggregateTargetsCheckBox.Name = "aggregateTargetsCheckBox";
       this.aggregateTargetsCheckBox.Size = new System.Drawing.Size(74, 17);
@@ -245 +247 @@
       // byTargetTabPage
       // 
+      this.byTargetTabPage.Controls.Add(this.relativeOrAbsoluteComboBox);
       this.byTargetTabPage.Controls.Add(this.targetChart);
+      this.byTargetTabPage.Controls.Add(this.markerCheckBox);
       this.byTargetTabPage.Controls.Add(this.boundShadingCheckBox);
       this.byTargetTabPage.Controls.Add(this.targetLogScalingCheckBox);
@@ -260 +264 @@
       this.byTargetTabPage.Text = "Performance by Target";
       this.byTargetTabPage.UseVisualStyleBackColor = true;
+      // 
+      // relativeOrAbsoluteComboBox
+      // 
+      this.relativeOrAbsoluteComboBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
+      this.relativeOrAbsoluteComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
+      this.relativeOrAbsoluteComboBox.FormattingEnabled = true;
+      this.relativeOrAbsoluteComboBox.Items.AddRange(new object[] {
+            "relative",
+            "absolute"});
+      this.relativeOrAbsoluteComboBox.Location = new System.Drawing.Point(269, 8);
+      this.relativeOrAbsoluteComboBox.Name = "relativeOrAbsoluteComboBox";
+      this.relativeOrAbsoluteComboBox.Size = new System.Drawing.Size(99, 21);
+      this.relativeOrAbsoluteComboBox.TabIndex = 8;
+      this.relativeOrAbsoluteComboBox.SelectedIndexChanged += new System.EventHandler(this.relativeOrAbsoluteComboBox_SelectedIndexChanged);
       // 
       // targetChart
@@ -291 +309 @@
       this.targetChart.MouseMove += new System.Windows.Forms.MouseEventHandler(this.chart_MouseMove);
       // 
+      // markerCheckBox
+      // 
+      this.markerCheckBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Bottom | System.Windows.Forms.AnchorStyles.Left)));
+      this.markerCheckBox.AutoSize = true;
+      this.markerCheckBox.Location = new System.Drawing.Point(229, 364);
+      this.markerCheckBox.Name = "markerCheckBox";
+      this.markerCheckBox.Size = new System.Drawing.Size(91, 17);
+      this.markerCheckBox.TabIndex = 6;
+      this.markerCheckBox.Text = "show markers";
+      this.markerCheckBox.UseVisualStyleBackColor = true;
+      this.markerCheckBox.CheckedChanged += new System.EventHandler(this.markerCheckBox_CheckedChanged);
+      // 
       // boundShadingCheckBox
       // 
@@ -309 +339 @@
       this.byCostTabPage.Controls.Add(this.byCostViewHost);
       this.byCostTabPage.Controls.Add(this.budgetLogScalingCheckBox);
-      this.byCostTabPage.Controls.Add(this.eachOrAllBudgetsCheckBox);
+      this.byCostTabPage.Controls.Add(this.aggregateBudgetsCheckBox);
       this.byCostTabPage.Controls.Add(this.generateBudgetsButton);
       this.byCostTabPage.Controls.Add(this.budgetsLabel);
@@ -350 +380 @@
       this.budgetLogScalingCheckBox.CheckedChanged += new System.EventHandler(this.logScalingCheckBox_CheckedChanged);
       // 
-      // eachOrAllBudgetsCheckBox
-      // 
-      this.eachOrAllBudgetsCheckBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
-      this.eachOrAllBudgetsCheckBox.AutoSize = true;
-      this.eachOrAllBudgetsCheckBox.Location = new System.Drawing.Point(414, 10);
-      this.eachOrAllBudgetsCheckBox.Name = "eachOrAllBudgetsCheckBox";
-      this.eachOrAllBudgetsCheckBox.Size = new System.Drawing.Size(36, 17);
-      this.eachOrAllBudgetsCheckBox.TabIndex = 10;
-      this.eachOrAllBudgetsCheckBox.Text = "all";
-      this.eachOrAllBudgetsCheckBox.UseVisualStyleBackColor = true;
-      this.eachOrAllBudgetsCheckBox.CheckedChanged += new System.EventHandler(this.eachOrAllBudgetsCheckBox_CheckedChanged);
+      // aggregateBudgetsCheckBox
+      // 
+      this.aggregateBudgetsCheckBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
+      this.aggregateBudgetsCheckBox.AutoSize = true;
+      this.aggregateBudgetsCheckBox.Checked = true;
+      this.aggregateBudgetsCheckBox.CheckState = System.Windows.Forms.CheckState.Checked;
+      this.aggregateBudgetsCheckBox.Location = new System.Drawing.Point(374, 10);
+      this.aggregateBudgetsCheckBox.Name = "aggregateBudgetsCheckBox";
+      this.aggregateBudgetsCheckBox.Size = new System.Drawing.Size(74, 17);
+      this.aggregateBudgetsCheckBox.TabIndex = 10;
+      this.aggregateBudgetsCheckBox.Text = "aggregate";
+      this.aggregateBudgetsCheckBox.UseVisualStyleBackColor = true;
+      this.aggregateBudgetsCheckBox.CheckedChanged += new System.EventHandler(this.aggregateBudgetsCheckBox_CheckedChanged);
       // 
       // generateBudgetsButton
@@ -464 +496 @@
     private System.Windows.Forms.TabPage byCostTabPage;
     private System.Windows.Forms.TabPage byTableTabPage;
-    private System.Windows.Forms.CheckBox eachOrAllBudgetsCheckBox;
+    private System.Windows.Forms.CheckBox aggregateBudgetsCheckBox;
     private System.Windows.Forms.Button generateBudgetsButton;
     private System.Windows.Forms.CheckBox budgetLogScalingCheckBox;
@@ -473 +505 @@
     private MainForm.WindowsForms.ViewHost byCostViewHost;
     private System.Windows.Forms.CheckBox boundShadingCheckBox;
+    private System.Windows.Forms.CheckBox markerCheckBox;
+    private System.Windows.Forms.ComboBox relativeOrAbsoluteComboBox;
   }
 }

branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs

@@ -30 +30 @@
 using HeuristicLab.Analysis;
 using HeuristicLab.Collections;
+using HeuristicLab.Core;
 using HeuristicLab.Core.Views;
 using HeuristicLab.Data;
@@ -79 +80 @@
     private double[] targets;
     private double[] budgets;
+    private bool targetsAreRelative = true;
+    private readonly BindingList<ProblemInstance> problems;
 
     private bool suppressUpdates;
@@ -103 +106 @@
       };
       byCostViewHost.Content = byCostDataTable;
+      suppressUpdates = true;
+      relativeOrAbsoluteComboBox.SelectedItem = targetsAreRelative ? "relative" : "absolute";
+      problems = new BindingList<ProblemInstance>();
+      problemComboBox.DataSource = new BindingSource() { DataSource = problems };
+      problemComboBox.DataBindings.DefaultDataSourceUpdateMode = DataSourceUpdateMode.OnPropertyChanged;
       suppressUpdates = false;
     }
@@ -223 +231 @@
       }
 
-      var problems = new HashSet<ProblemDescription>();
-      foreach (var run in Content) {
-        problems.Add(new ProblemDescription(run));
-      }
-
-      var problemTypesDifferent = problems.Select(x => x.ProblemType).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
-      var problemNamesDifferent = problems.Select(x => x.ProblemName).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
-      var evaluatorDifferent = problems.Select(x => x.Evaluator).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
-      var maximizationDifferent = problems.Select(x => x.Maximization).Distinct().Count() > 1;
+      var table = (string)dataTableComboBox.SelectedItem;
+      var problemDict = CalculateBestTargetPerProblemInstance(table);
+
+      var problemTypesDifferent = problemDict.Keys.Select(x => x.ProblemType).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
+      var problemNamesDifferent = problemDict.Keys.Select(x => x.ProblemName).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
+      var evaluatorDifferent = problemDict.Keys.Select(x => x.Evaluator).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
+      var maximizationDifferent = problemDict.Keys.Select(x => x.Maximization).Distinct().Count() > 1;
       var allEqual = !problemTypesDifferent && !problemNamesDifferent && !evaluatorDifferent && !maximizationDifferent;
 
-      var selectedProblemItem = (ProblemDescription)problemComboBox.SelectedItem;
-      problemComboBox.Items.Clear();
-      problemComboBox.Items.Add(ProblemDescription.MatchAll);
-      if (selectedProblemItem == null || selectedProblemItem == ProblemDescription.MatchAll)
-        problemComboBox.SelectedIndex = 0;
-      foreach (var prob in problems.OrderBy(x => x.ToString()).ToList()) {
-        prob.DisplayProblemType = problemTypesDifferent;
-        prob.DisplayProblemName = problemNamesDifferent || allEqual;
-        prob.DisplayEvaluator = evaluatorDifferent;
-        prob.DisplayMaximization = maximizationDifferent;
-        problemComboBox.Items.Add(prob);
-        if (prob.Equals(selectedProblemItem)) problemComboBox.SelectedItem = prob;
+      var selectedProblemItem = (ProblemInstance)problemComboBox.SelectedItem;
+      problems.Clear();
+      problems.Add(ProblemInstance.MatchAll);
+      if (problems[0].Equals(selectedProblemItem)) problemComboBox.SelectedItem = problems[0];
+      foreach (var p in problemDict.ToList()) {
+        p.Key.BestKnownQuality = p.Value;
+        p.Key.DisplayProblemType = problemTypesDifferent;
+        p.Key.DisplayProblemName = problemNamesDifferent || allEqual;
+        p.Key.DisplayEvaluator = evaluatorDifferent;
+        p.Key.DisplayMaximization = maximizationDifferent;
+        problems.Add(p.Key);
+        if (p.Key.Equals(selectedProblemItem)) problemComboBox.SelectedItem = p.Key;
       }
       SetEnabledStateOfControls();
@@ -276 +282 @@
     }
 
-    private Dictionary<string, Dictionary<ProblemDescription, Tuple<double, List<IRun>>>> GroupRuns() {
-      var groupedRuns = new Dictionary<string, Dictionary<ProblemDescription, Tuple<double, List<IRun>>>>();
+    private Dictionary<string, Dictionary<ProblemInstance, List<IRun>>> GroupRuns() {
+      var groupedRuns = new Dictionary<string, Dictionary<ProblemInstance, List<IRun>>>();
 
       var table = (string)dataTableComboBox.SelectedItem;
@@ -285 +291 @@
       if (string.IsNullOrEmpty(selectedGroup)) return groupedRuns;
 
-      var selectedProblem = (ProblemDescription)problemComboBox.SelectedItem;
+      var selectedProblem = (ProblemInstance)problemComboBox.SelectedItem;
       if (selectedProblem == null) return groupedRuns;
-
-      var targetsPerProblem = CalculateBestTargetPerProblemInstance(table);
-
+      
       foreach (var x in (from r in Content
                          where (selectedGroup == AllRuns || r.Parameters.ContainsKey(selectedGroup))
@@ -295 +299 @@
                            && r.Results.ContainsKey(table)
                            && r.Visible
-                         group r by selectedGroup == AllRuns ? AllRuns : r.Parameters[selectedGroup].ToString() into g
+                         let key = selectedGroup == AllRuns ? AllRuns : r.Parameters[selectedGroup].ToString()
+                         group r by key into g
                          select Tuple.Create(g.Key, g.ToList()))) {
-        var pDict = new Dictionary<ProblemDescription, Tuple<double, List<IRun>>>();
-        foreach (var y in (from r in x.Item2
-                           let pd = new ProblemDescription(r)
-                           group r by pd into g
-                           select Tuple.Create(g.Key, g.ToList()))) {
-          pDict[y.Item1] = Tuple.Create(targetsPerProblem[y.Item1], y.Item2);
-        }
-        groupedRuns[x.Item1] = pDict;
+        var pDict = problems.ToDictionary(r => r, _ => new List<IRun>());
+        foreach (var y in x.Item2) {
+          var pd = new ProblemInstance(y);
+          List<IRun> l;
+          if (pDict.TryGetValue(pd, out l)) l.Add(y);
+        }
+        groupedRuns[x.Item1] = pDict.Where(a => a.Value.Count > 0).ToDictionary(a => a.Key, a => a.Value);
       }
 
@@ -326 +330 @@
 
       var xAxisTitles = new HashSet<string>();
+
+      // hits describes the number of target hits at a certain time for a certain group
+      var hits = new Dictionary<string, SortedList<double, int>>();
+      // misses describes the number of target misses after a certain time for a certain group
+      // for instance when a run ends, but has not achieved all targets, misses describes
+      // how many targets have been left open at the point when the run ended
+      var misses = new Dictionary<string, SortedList<double, int>>();
+
+      var aggregate = aggregateTargetsCheckBox.Checked;
+      double minEff = double.MaxValue, maxEff = double.MinValue;
+      var noRuns = 0;
+      foreach (var group in groupedRuns) {
+        SortedList<double, int> epdfHits = null, epdfMisses = null;
+        if (aggregate) {
+          hits[group.Key] = epdfHits = new SortedList<double, int>();
+          misses[group.Key] = epdfMisses = new SortedList<double, int>();
+        }
+        foreach (var problem in group.Value) {
+          var max = problem.Key.IsMaximization();
+          var absTargets = GetAbsoluteTargets(problem.Key).ToArray();
+          foreach (var run in problem.Value) {
+            noRuns++;
+            var resultsTable = (IndexedDataTable<double>)run.Results[table];
+            xAxisTitles.Add(resultsTable.VisualProperties.XAxisTitle);
+
+            var efforts = absTargets.Select(t => GetEffortToHitTarget(resultsTable.Rows.First().Values, t, max)).ToArray();
+            minEff = Math.Min(minEff, efforts.Min(x => x.Item2));
+            maxEff = Math.Max(maxEff, efforts.Max(x => x.Item2));
+            for (var idx = 0; idx < efforts.Length; idx++) {
+              var e = efforts[idx];
+              if (!aggregate) {
+                var key = group.Key + "@" + (targetsAreRelative
+                  ? (targets[idx] * 100).ToString(CultureInfo.CurrentCulture.NumberFormat) + "%"
+                  : targets[idx].ToString(CultureInfo.CurrentCulture.NumberFormat));
+                if (!hits.TryGetValue(key, out epdfHits))
+                  hits[key] = epdfHits = new SortedList<double, int>();
+                if (!misses.TryGetValue(key, out epdfMisses))
+                  misses[key] = epdfMisses = new SortedList<double, int>();
+              }
+              var list = e.Item1 ? epdfHits : epdfMisses;
+              int v;
+              if (list.TryGetValue(e.Item2, out v))
+                list[e.Item2] = v + 1;
+              else list[e.Item2] = 1;
+            }
+          }
+        }
+      }
+
+      UpdateTargetChartAxisXBounds(minEff, maxEff);
+
+      DrawTargetsEcdf(hits, misses, noRuns);
+
+      if (targets.Length == 1) {
+        if (targetsAreRelative)
+          targetChart.ChartAreas[0].AxisY.Title = "Probability to be " + (targets[0] * 100) + "% worse than best";
+        else targetChart.ChartAreas[0].AxisY.Title = "Probability to reach at least a fitness of " + targets[0];
+      } else targetChart.ChartAreas[0].AxisY.Title = "Proportion of reached targets";
+      targetChart.ChartAreas[0].AxisX.Title = string.Join(" / ", xAxisTitles);
+      targetChart.ChartAreas[0].AxisX.IsLogarithmic = CanDisplayLogarithmic();
+      targetChart.ChartAreas[0].CursorY.Interval = 0.05;
+
+      UpdateErtTables(groupedRuns);
+    }
+
+    private void DrawTargetsEcdf(Dictionary<string, SortedList<double, int>> hits, Dictionary<string, SortedList<double, int>> misses, int noRuns) {
       var colorCount = 0;
       var lineStyleCount = 0;
-
-      // if the group contains multiple different problem instances we want to use the
-      // minimal maximal observed effort otherwise we run into situations where we don't
-      // have data for a certain problem instance anymore this is a special case when
-      // aggregating over multiple problem instances      
-      var maxEfforts = new Dictionary<ProblemDescription, double>();
-      double minEff = double.MaxValue, maxEff = double.MinValue;
-      foreach (var group in groupedRuns) {
-        foreach (var problem in group.Value) {
-          double problemSpecificMaxEff;
-          if (!maxEfforts.TryGetValue(problem.Key, out problemSpecificMaxEff)) {
-            problemSpecificMaxEff = 0;
-          }
-          var bestKnownTarget = problem.Value.Item1;
-          var max = problem.Key.IsMaximization();
-          var worstTarget = (max ? (1 - targets.Max()) : (1 + targets.Max())) * bestKnownTarget;
-          var bestTarget = (max ? (1 - targets.Min()) : (1 + targets.Min())) * bestKnownTarget;
-          foreach (var run in problem.Value.Item2) {
-            var row = ((IndexedDataTable<double>)run.Results[table]).Rows.First().Values;
-            var a = row.FirstOrDefault(x => max ? x.Item2 >= worstTarget : x.Item2 <= worstTarget);
-            var b = row.FirstOrDefault(x => max ? x.Item2 >= bestTarget : x.Item2 <= bestTarget);
-            var firstEff = (a == default(Tuple<double, double>)) ? row.Last().Item1 : a.Item1;
-            var lastEff = (b == default(Tuple<double, double>)) ? row.Last().Item1 : b.Item1;
-            if (minEff > firstEff) minEff = firstEff;
-            if (maxEff < lastEff) maxEff = lastEff;
-            if (problemSpecificMaxEff < lastEff) problemSpecificMaxEff = lastEff;
-          }
-          maxEfforts[problem.Key] = problemSpecificMaxEff;
-        }
-      }
-      maxEff = Math.Min(maxEff, maxEfforts.Values.Min());
-
+      
+      var showMarkers = markerCheckBox.Checked;
+      foreach (var list in hits) {
+        var row = new Series(list.Key) {
+          ChartType = SeriesChartType.StepLine,
+          BorderWidth = 3,
+          Color = colors[colorCount],
+          BorderDashStyle = lineStyles[lineStyleCount],
+        };
+        var rowShade = new Series(list.Key + "-range") {
+          IsVisibleInLegend = false,
+          ChartType = SeriesChartType.Range,
+          Color = Color.FromArgb(32, colors[colorCount]),
+          YValuesPerPoint = 2
+        };
+
+        var ecdf = 0.0;
+        var missedecdf = 0.0;
+        var iter = misses[list.Key].GetEnumerator();
+        var moreMisses = iter.MoveNext();
+        var totalTargets = noRuns;
+        if (aggregateTargetsCheckBox.Checked) totalTargets *= targets.Length;
+        var movingTargets = totalTargets;
+        var labelPrinted = false;
+        foreach (var h in list.Value) {
+          var prevmissedecdf = missedecdf;
+          while (moreMisses && iter.Current.Key <= h.Key) {
+            if (!labelPrinted && row.Points.Count > 0) {
+              var point = row.Points.Last();
+              point.Label = row.Name;
+              point.MarkerStyle = MarkerStyle.Cross;
+              point.MarkerBorderWidth = 1;
+              point.MarkerSize = 10;
+              labelPrinted = true;
+              rowShade.Points.Add(new DataPoint(point.XValue, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
+            }
+            missedecdf += iter.Current.Value;
+            movingTargets -= iter.Current.Value;
+            if (row.Points.Count > 0 && row.Points.Last().XValue == iter.Current.Key) {
+              row.Points.Last().SetValueY(ecdf / movingTargets);
+              row.Points.Last().Color = Color.FromArgb(255 - (int)Math.Floor(255 * (prevmissedecdf / totalTargets)), colors[colorCount]);
+            } else {
+              var dp = new DataPoint(iter.Current.Key, ecdf / movingTargets) {
+                Color = Color.FromArgb(255 - (int)Math.Floor(255 * (prevmissedecdf / totalTargets)), colors[colorCount])
+              };
+              if (showMarkers) {
+                dp.MarkerStyle = MarkerStyle.Circle;
+                dp.MarkerBorderWidth = 1;
+                dp.MarkerSize = 5;
+              }
+              row.Points.Add(dp);
+              prevmissedecdf = missedecdf;
+            }
+            if (boundShadingCheckBox.Checked) {
+              if (rowShade.Points.Count > 0 && rowShade.Points.Last().XValue == iter.Current.Key)
+                rowShade.Points.Last().SetValueY(ecdf / totalTargets, (ecdf + missedecdf) / totalTargets);
+              else rowShade.Points.Add(new DataPoint(iter.Current.Key, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
+            }
+            moreMisses = iter.MoveNext();
+            if (!labelPrinted) {
+              var point = row.Points.Last();
+              point.Label = row.Name;
+              point.MarkerStyle = MarkerStyle.Cross;
+              point.MarkerBorderWidth = 1;
+              point.MarkerSize = 10;
+              labelPrinted = true;
+            }
+          }
+          ecdf += h.Value;
+          if (row.Points.Count > 0 && row.Points.Last().XValue == h.Key) {
+            row.Points.Last().SetValueY(ecdf / movingTargets);
+            row.Points.Last().Color = Color.FromArgb(255 - (int)Math.Floor(255 * (missedecdf / totalTargets)), colors[colorCount]);
+          } else {
+            var dp = new DataPoint(h.Key, ecdf / movingTargets) {
+              Color = Color.FromArgb(255 - (int)Math.Floor(255 * (missedecdf / totalTargets)), colors[colorCount])
+            };
+            if (showMarkers) {
+              dp.MarkerStyle = MarkerStyle.Circle;
+              dp.MarkerBorderWidth = 1;
+              dp.MarkerSize = 5;
+            }
+            row.Points.Add(dp);
+          }
+          if (missedecdf > 0 && boundShadingCheckBox.Checked) {
+            if (rowShade.Points.Count > 0 && rowShade.Points.Last().XValue == h.Key)
+              rowShade.Points.Last().SetValueY(ecdf / totalTargets, (ecdf + missedecdf) / totalTargets);
+            else rowShade.Points.Add(new DataPoint(h.Key, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
+          }
+        }
+
+        while (moreMisses) {
+          // if there are misses beyond the last hit we extend the shaded area
+          missedecdf += iter.Current.Value;
+          var dp = new DataPoint(iter.Current.Key, ecdf / movingTargets) {
+            Color = Color.FromArgb(255 - (int)Math.Floor(255 * (missedecdf / totalTargets)), colors[colorCount])
+          };
+          if (showMarkers) {
+            dp.MarkerStyle = MarkerStyle.Circle;
+            dp.MarkerBorderWidth = 1;
+            dp.MarkerSize = 5;
+          }
+          row.Points.Add(dp);
+          if (boundShadingCheckBox.Checked) {
+            rowShade.Points.Add(new DataPoint(iter.Current.Key, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
+          }
+          moreMisses = iter.MoveNext();
+
+          if (!labelPrinted && row.Points.Count > 0) {
+            var point = row.Points.Last();
+            point.Label = row.Name;
+            point.MarkerStyle = MarkerStyle.Cross;
+            point.MarkerBorderWidth = 1;
+            point.MarkerSize = 10;
+            labelPrinted = true;
+          }
+        }
+
+        if (!labelPrinted) {
+          var point = row.Points.Last();
+          point.Label = row.Name;
+          point.MarkerStyle = MarkerStyle.Cross;
+          point.MarkerBorderWidth = 1;
+          point.MarkerSize = 10;
+          rowShade.Points.Add(new DataPoint(point.XValue, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
+          labelPrinted = true;
+        }
+
+        ConfigureSeries(row);
+        targetChart.Series.Add(rowShade);
+        targetChart.Series.Add(row);
+
+        colorCount = (colorCount + 1) % colors.Length;
+        if (colorCount == 0) lineStyleCount = (lineStyleCount + 1) % lineStyles.Length;
+      }
+    }
+
+    private void UpdateTargetChartAxisXBounds(double minEff, double maxEff) {
       var minZeros = (int)Math.Floor(Math.Log10(minEff));
       var maxZeros = (int)Math.Floor(Math.Log10(maxEff));
@@ -372 +547 @@
       targetChart.ChartAreas[0].AxisX.Minimum = (double)axisMin;
       targetChart.ChartAreas[0].AxisX.Maximum = (double)axisMax;
-
-      foreach (var group in groupedRuns) {
-        // hits describes the number of target hits at a certain time for a certain group
-        var hits = new Dictionary<string, SortedList<double, int>>();
-        // misses describes the number of target misses after a certain time for a certain group
-        // for instance when a run ends, but has not achieved all targets, misses describes
-        // how many targets have been left open at the point when the run ended
-        var misses = new Dictionary<string, SortedList<double, int>>();
-        var maxLength = 0.0;
-
-        var noRuns = 0;
-        foreach (var problem in group.Value) {
-          foreach (var run in problem.Value.Item2) {
-            var resultsTable = (IndexedDataTable<double>)run.Results[table];
-            xAxisTitles.Add(resultsTable.VisualProperties.XAxisTitle);
-
-            if (aggregateTargetsCheckBox.Checked) {
-              var length = CalculateHitsForAllTargets(hits, misses, resultsTable.Rows.First(), problem.Key, group.Key, problem.Value.Item1, maxEff);
-              maxLength = Math.Max(length, maxLength);
-            } else {
-              CalculateHitsForEachTarget(hits, misses, resultsTable.Rows.First(), problem.Key, group.Key, problem.Value.Item1, maxEff);
-            }
-            noRuns++;
-          }
-        }
-        foreach (var list in hits) {
-          var row = new Series(list.Key) {
-            ChartType = SeriesChartType.StepLine,
-            BorderWidth = 2,
-            Color = colors[colorCount],
-            BorderDashStyle = lineStyles[lineStyleCount],
-          };
-          var rowShade = new Series(list.Key + "-range") {
-            IsVisibleInLegend = false,
-            ChartType = SeriesChartType.Range,
-            Color = Color.FromArgb(32, colors[colorCount])
-          };
-
-          var ecdf = 0.0;
-          var missedecdf = 0.0;
-          var iter = misses[list.Key].GetEnumerator();
-          var moreMisses = iter.MoveNext();
-          var totalTargets = noRuns;
-          if (aggregateTargetsCheckBox.Checked) totalTargets *= targets.Length;
-          var movingTargets = totalTargets;
-          foreach (var h in list.Value) {
-            while (moreMisses && iter.Current.Key <= h.Key) {
-              missedecdf += iter.Current.Value;
-              movingTargets -= iter.Current.Value;
-              if (row.Points.Count > 0 && row.Points.Last().XValue == iter.Current.Key)
-                row.Points.Last().SetValueY(ecdf / movingTargets);
-              else row.Points.AddXY(iter.Current.Key, ecdf / movingTargets);
-              if (boundShadingCheckBox.Checked) {
-                if (rowShade.Points.Count > 0 && rowShade.Points.Last().XValue == iter.Current.Key)
-                  rowShade.Points.Last().SetValueY(ecdf / totalTargets, (ecdf + missedecdf) / totalTargets);
-                else rowShade.Points.Add(new DataPoint(iter.Current.Key, new[] { ecdf / totalTargets, (ecdf + missedecdf) / totalTargets }));
-              }
-              moreMisses = iter.MoveNext();
-            }
-            ecdf += h.Value;
-            if (row.Points.Count > 0 && row.Points.Last().XValue == h.Key)
-              row.Points.Last().SetValueY(ecdf / movingTargets);
-            else row.Points.AddXY(h.Key, ecdf / movingTargets);
-            if (missedecdf > 0 && boundShadingCheckBox.Checked) {
-              if (rowShade.Points.Count > 0 && rowShade.Points.Last().XValue == h.Key)
-                rowShade.Points.Last().SetValueY(ecdf / totalTargets, (ecdf + missedecdf) / totalTargets);
-              else rowShade.Points.Add(new DataPoint(h.Key, new[] { ecdf / totalTargets, (ecdf + missedecdf) / totalTargets }));
-            }
-          }
-
-          while (moreMisses) {
-            // if there are misses beyond the last hit we extend the shaded area
-            missedecdf += iter.Current.Value;
-            //movingTargets -= iter.Current.Value;
-            if (row.Points.Count > 0 && row.Points.Last().XValue == iter.Current.Key)
-              row.Points.Last().SetValueY(ecdf / movingTargets);
-            else row.Points.AddXY(iter.Current.Key, ecdf / movingTargets);
-            if (boundShadingCheckBox.Checked) {
-              if (rowShade.Points.Count > 0 && rowShade.Points.Last().XValue == iter.Current.Key)
-                rowShade.Points.Last().SetValueY(ecdf / totalTargets, (ecdf + missedecdf) / totalTargets);
-              else rowShade.Points.Add(new DataPoint(iter.Current.Key, new[] { ecdf / totalTargets, (ecdf + missedecdf) / totalTargets }));
-            }
-            moreMisses = iter.MoveNext();
-          }
-
-          if (maxLength > 0 && (row.Points.Count == 0 || row.Points.Last().XValue < maxLength))
-            row.Points.AddXY(maxLength, ecdf / movingTargets);
-
-          if (row.Points.Count > 0) {
-            var point = row.Points.Last();
-            point.Label = row.Name;
-            point.MarkerStyle = MarkerStyle.Cross;
-            point.MarkerBorderWidth = 1;
-          }
-
-          ConfigureSeries(row);
-          targetChart.Series.Add(rowShade);
-          targetChart.Series.Add(row);
-        }
-        colorCount = (colorCount + 1) % colors.Length;
-        if (colorCount == 0) lineStyleCount = (lineStyleCount + 1) % lineStyles.Length;
-      }
-
-      if (targets.Length == 1)
-        targetChart.ChartAreas[0].AxisY.Title = "Probability to be " + (targets[0] * 100) + "% worse than best";
-      else targetChart.ChartAreas[0].AxisY.Title = "Proportion of reached targets";
-      targetChart.ChartAreas[0].AxisX.Title = string.Join(" / ", xAxisTitles);
-      targetChart.ChartAreas[0].AxisX.IsLogarithmic = CanDisplayLogarithmic();
-      targetChart.ChartAreas[0].CursorY.Interval = 0.05;
-      UpdateErtTables(groupedRuns);
+    }
+
+    private IEnumerable<double> GetAbsoluteTargets(ProblemInstance pInstance) {
+      if (!targetsAreRelative) return targets;
+      var maximization = pInstance.IsMaximization();
+      var bestKnown = pInstance.BestKnownQuality;
+      if (double.IsNaN(bestKnown)) throw new ArgumentException("Problem instance does not have a defined best - known quality.");
+      IEnumerable<double> tmp = null;
+      if (bestKnown > 0) {
+        tmp = targets.Select(x => (maximization ? (1 - x) : (1 + x)) * bestKnown);
+      } else if (bestKnown < 0) {
+        tmp = targets.Select(x => (!maximization ? (1 - x) : (1 + x)) * bestKnown);
+      } else {
+        // relative to 0 is impossible
+        tmp = targets;
+      }
+      return tmp;
+    }
+
+    private double[] GetAbsoluteTargetsWorstToBest(ProblemInstance pInstance) {
+      if (double.IsNaN(pInstance.BestKnownQuality)) throw new ArgumentException("Problem instance does not have a defined best-known quality.");
+      var absTargets = GetAbsoluteTargets(pInstance);
+      return (pInstance.IsMaximization()
+        ? absTargets.OrderBy(x => x) : absTargets.OrderByDescending(x => x)).ToArray();
     }
 
     private void GenerateDefaultTargets() {
-      targets = new[] { 0.1, 0.05, 0.02, 0.01, 0 };
+      targets = new[] { 0.1, 0.095, 0.09, 0.085, 0.08, 0.075, 0.07, 0.065, 0.06, 0.055, 0.05, 0.045, 0.04, 0.035, 0.03, 0.025, 0.02, 0.015, 0.01, 0.005, 0 };
       suppressTargetsEvents = true;
       targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100)) + "%";
@@ -491 +580 @@
     }
 
-    private void CalculateHitsForEachTarget(Dictionary<string, SortedList<double, int>> hits,
-                                            Dictionary<string, SortedList<double, int>> misses,
-                                            IndexedDataRow<double> row, ProblemDescription problem,
-                                            string group, double bestTarget, double maxEffort) {
-      foreach (var t in targets.Select(x => Tuple.Create((problem.IsMaximization() ? (1 - x) : (1 + x)) * bestTarget, x))) {
-        var l = t.Item1;
-        var key = group + "_" + (t.Item2 * 100) + "%_" + l;
-        if (!hits.ContainsKey(key)) {
-          hits.Add(key, new SortedList<double, int>());
-          misses.Add(key, new SortedList<double, int>());
-        }
-        var hit = false;
-        foreach (var v in row.Values) {
-          if (v.Item1 > maxEffort) break;
-          if (problem.IsMaximization() && v.Item2 >= l || !problem.IsMaximization() && v.Item2 <= l) {
-            if (hits[key].ContainsKey(v.Item1))
-              hits[key][v.Item1]++;
-            else hits[key][v.Item1] = 1;
-            hit = true;
-            break;
-          }
-        }
-        if (!hit) {
-          var max = Math.Min(row.Values.Last().Item1, maxEffort);
-          if (misses[key].ContainsKey(max))
-            misses[key][max]++;
-          else misses[key][max] = 1;
-        }
-      }
-    }
-
-    private double CalculateHitsForAllTargets(Dictionary<string, SortedList<double, int>> hits,
-                                              Dictionary<string, SortedList<double, int>> misses,
-                                              IndexedDataRow<double> row, ProblemDescription problem,
-                                              string group, double bestTarget, double maxEffort) {
-      var values = row.Values;
-      if (!hits.ContainsKey(group)) {
-        hits.Add(group, new SortedList<double, int>());
-        misses.Add(group, new SortedList<double, int>());
-      }
-
-      var i = 0;
-      var j = 0;
-      while (i < targets.Length && j < values.Count) {
-        var target = (problem.IsMaximization() ? (1 - targets[i]) : (1 + targets[i])) * bestTarget;
-        var current = values[j];
-        if (current.Item1 > maxEffort) break;
-        if (problem.IsMaximization() && current.Item2 >= target
-          || !problem.IsMaximization() && current.Item2 <= target) {
-          if (hits[group].ContainsKey(current.Item1)) hits[group][current.Item1]++;
-          else hits[group][current.Item1] = 1;
-          i++;
-        } else {
-          j++;
-        }
-      }
-      if (j == values.Count) j--;
-      var effort = Math.Min(values[j].Item1, maxEffort);
-      if (i < targets.Length) {
-        if (misses[group].ContainsKey(effort))
-          misses[group][effort] += targets.Length - i;
-        else misses[group][effort] = targets.Length - i;
-      }
-      return effort;
-    }
-
-    private void UpdateErtTables(Dictionary<string, Dictionary<ProblemDescription, Tuple<double, List<IRun>>>> groupedRuns) {
+    private Tuple<bool, double> GetEffortToHitTarget(
+        IEnumerable<Tuple<double, double>> convergenceGraph,
+        double absTarget, bool maximization) {
+      var hit = false;
+      var effort = double.NaN;
+      foreach (var dent in convergenceGraph) {
+        effort = dent.Item1;
+        hit = maximization && dent.Item2 >= absTarget || !maximization && dent.Item2 <= absTarget;
+        if (hit) break;
+      }
+      if (double.IsNaN(effort)) throw new ArgumentException("Convergence graph is empty.", "convergenceGraph");
+      return Tuple.Create(hit, effort);
+    }
+
+    private void UpdateErtTables(Dictionary<string, Dictionary<ProblemInstance, List<IRun>>> groupedRuns) {
       ertTableView.Content = null;
       var columns = 1 + targets.Length + 1;
@@ -565 +602 @@
       var tableName = (string)dataTableComboBox.SelectedItem;
       if (string.IsNullOrEmpty(tableName)) return;
-
-      var targetsPerProblem = CalculateBestTargetPerProblemInstance(tableName);
-
-      var colNames = new string[columns];
-      colNames[0] = colNames[colNames.Length - 1] = string.Empty;
-      for (var i = 0; i < targets.Length; i++) {
-        colNames[i + 1] = targets[i].ToString("0.0%");
-      }
+      
       var problems = groupedRuns.SelectMany(x => x.Value.Keys).Distinct().ToList();
 
       foreach (var problem in problems) {
+        var max = problem.IsMaximization();
         matrix[rowCount, 0] = problem.ToString();
-        for (var i = 0; i < targets.Length; i++) {
-          matrix[rowCount, i + 1] = (targetsPerProblem[problem] * (problem.IsMaximization() ? (1 - targets[i]) : (1 + targets[i]))).ToString(CultureInfo.CurrentCulture.NumberFormat);
+        var absTargets = GetAbsoluteTargetsWorstToBest(problem);
+        for (var i = 0; i < absTargets.Length; i++) {
+          matrix[rowCount, i + 1] = absTargets[i].ToString(CultureInfo.CurrentCulture.NumberFormat);
         }
         matrix[rowCount, columns - 1] = "#succ";
@@ -590 +622 @@
             continue;
           }
-          var runs = group.Value[problem].Item2;
-          ErtCalculationResult result = default(ErtCalculationResult);
-          for (var i = 0; i < targets.Length; i++) {
-            result = ExpectedRuntimeHelper.CalculateErt(runs, tableName, (problem.IsMaximization() ? (1 - targets[i]) : (1 + targets[i])) * group.Value[problem].Item1, problem.IsMaximization());
+          var runs = group.Value[problem];
+          var result = default(ErtCalculationResult);
+          for (var i = 0; i < absTargets.Length; i++) {
+            result = ExpectedRuntimeHelper.CalculateErt(runs, tableName, absTargets[i], max);
             matrix[rowCount, i + 1] = result.ToString();
           }
@@ -600 +632 @@
         }
       }
-      ertTableView.Content = new StringMatrix(matrix) { ColumnNames = colNames };
+      ertTableView.Content = new StringMatrix(matrix);
       ertTableView.DataGridView.AutoResizeColumns(DataGridViewAutoSizeColumnsMode.AllCells);
     }
@@ -621 +653 @@
       var colorCount = 0;
       var lineStyleCount = 0;
-
-      var targetsPerProblem = CalculateBestTargetPerProblemInstance((string)dataTableComboBox.SelectedItem);
-
+      
       foreach (var group in groupedRuns) {
         var hits = new Dictionary<string, SortedList<double, double>>();
 
         foreach (var problem in group.Value) {
-          foreach (var run in problem.Value.Item2) {
+          foreach (var run in problem.Value) {
             var resultsTable = (IndexedDataTable<double>)run.Results[table];
 
-            if (eachOrAllBudgetsCheckBox.Checked) {
-              CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), group.Value.Count, problem.Key, group.Key, problem.Value.Item2.Count, targetsPerProblem[problem.Key]);
+            if (aggregateBudgetsCheckBox.Checked) {
+              CalculateHitsForAllBudgets(hits, resultsTable.Rows.First(), group.Value.Count, problem.Key, group.Key, problem.Value.Count);
             } else {
-              CalculateHitsForAllBudgets(hits, resultsTable.Rows.First(), group.Value.Count, problem.Key, group.Key, problem.Value.Item2.Count, targetsPerProblem[problem.Key]);
+              CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), group.Value.Count, problem.Key, group.Key, problem.Value.Count);
             }
           }
@@ -667 +697 @@
 
     private void GenerateDefaultBudgets(string table) {
-      var runs = GroupRuns().SelectMany(x => x.Value.Values).SelectMany(x => x.Item2).ToList();
+      var runs = GroupRuns().SelectMany(x => x.Value.Values).SelectMany(x => x).ToList();
       var min = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Min()).Min();
       var max = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Max()).Max();
-
-      var maxMagnitude = (int)Math.Ceiling(Math.Log10(max));
-      var minMagnitude = (int)Math.Floor(Math.Log10(min));
-      if (maxMagnitude - minMagnitude >= 3) {
-        budgets = new double[maxMagnitude - minMagnitude];
-        for (var i = minMagnitude; i < maxMagnitude; i++) {
-          budgets[i - minMagnitude] = Math.Pow(10, i);
-        }
-      } else {
-        var range = max - min;
-        budgets = Enumerable.Range(0, 6).Select(x => min + (x / 5.0) * range).ToArray();
-      }
+      var points = 20;
+      budgets = Enumerable.Range(1, points).Select(x => min + (x / (double)points) * (max - min)).ToArray();
       suppressBudgetsEvents = true;
       budgetsTextBox.Text = string.Join(" ; ", budgets);
@@ -687 +707 @@
     }
 
-    private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, ProblemDescription problem, string groupName, int problemCount, double bestTarget) {
+    private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, ProblemInstance problem, string groupName, int problemCount) {
+      var max = problem.IsMaximization();
       foreach (var b in budgets) {
         var key = groupName + "-" + b;
@@ -697 +718 @@
             if (prev == null && v.Item1 != b) break;
             var tgt = ((prev == null || v.Item1 == b) ? v.Item2 : prev.Item2);
-            tgt = problem.IsMaximization() ? bestTarget / tgt : tgt / bestTarget;
-            if (hits[key].ContainsKey(tgt))
-              hits[key][tgt] += 1.0 / (groupCount * problemCount);
-            else hits[key][tgt] = 1.0 / (groupCount * problemCount);
+            var relTgt = CalculateRelativeDifference(max, problem.BestKnownQuality, tgt) + 1;
+            if (hits[key].ContainsKey(relTgt))
+              hits[key][relTgt] += 1.0 / (groupCount * problemCount);
+            else hits[key][relTgt] = 1.0 / (groupCount * problemCount);
             break;
           }
@@ -709 +730 @@
     }
 
-    private void CalculateHitsForAllBudgets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, ProblemDescription problem, string groupName, int problemCount, double bestTarget) {
+    private void CalculateHitsForAllBudgets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, ProblemInstance problem, string groupName, int problemCount) {
       var values = row.Values;
       if (!hits.ContainsKey(groupName)) hits.Add(groupName, new SortedList<double, double>());
@@ -716 +737 @@
       var j = 0;
       Tuple<double, double> prev = null;
+      var max = problem.IsMaximization();
       while (i < budgets.Length && j < values.Count) {
         var current = values[j];
@@ -721 +743 @@
           if (prev != null || current.Item1 == budgets[i]) {
             var tgt = (prev == null || current.Item1 == budgets[i]) ? current.Item2 : prev.Item2;
-            tgt = problem.IsMaximization() ? bestTarget / tgt : tgt / bestTarget;
-            if (!hits[groupName].ContainsKey(tgt)) hits[groupName][tgt] = 0;
-            hits[groupName][tgt] += 1.0 / (groupCount * problemCount * budgets.Length);
+            var relTgt = CalculateRelativeDifference(max, problem.BestKnownQuality, tgt) + 1;
+            if (!hits[groupName].ContainsKey(relTgt)) hits[groupName][relTgt] = 0;
+            hits[groupName][relTgt] += 1.0 / (groupCount * problemCount * budgets.Length);
           }
           i++;
@@ -732 +754 @@
       }
       var lastTgt = values.Last().Item2;
-      lastTgt = problem.IsMaximization() ? bestTarget / lastTgt : lastTgt / bestTarget;
-      if (i < budgets.Length && !hits[groupName].ContainsKey(lastTgt)) hits[groupName][lastTgt] = 0;
+      var lastRelTgt = CalculateRelativeDifference(max, problem.BestKnownQuality, lastTgt) + 1;
+      if (i < budgets.Length && !hits[groupName].ContainsKey(lastRelTgt)) hits[groupName][lastRelTgt] = 0;
       while (i < budgets.Length) {
-        hits[groupName][lastTgt] += 1.0 / (groupCount * problemCount * budgets.Length);
+        hits[groupName][lastRelTgt] += 1.0 / (groupCount * problemCount * budgets.Length);
         i++;
       }
@@ -756 +778 @@
       if (dataTableComboBox.SelectedIndex >= 0)
         GenerateDefaultBudgets((string)dataTableComboBox.SelectedItem);
+      UpdateBestKnownQualities();
       UpdateRuns();
       SetEnabledStateOfControls();
@@ -782 +805 @@
           return;
         }
-        targetList.Add(t / 100);
+        if (targetsAreRelative)
+          targetList.Add(t / 100);
+        else targetList.Add(t);
       }
       if (targetList.Count == 0) {
@@ -791 +816 @@
       e.Cancel = false;
       errorProvider.SetError(targetsTextBox, null);
-      targets = targetList.Select(x => (double)x).OrderByDescending(x => x).ToArray();
+      targets = targetsAreRelative ? targetList.Select(x => (double)x).OrderByDescending(x => x).ToArray() : targetList.Select(x => (double)x).ToArray();
       suppressTargetsEvents = true;
-      try { targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100)) + "%"; } finally { suppressTargetsEvents = false; }
+      try {
+        if (targetsAreRelative)
+          targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100)) + "%";
+        else targetsTextBox.Text = string.Join(" ; ", targets);
+      } finally { suppressTargetsEvents = false; }
 
       UpdateResultsByTarget();
@@ -804 +833 @@
         UpdateResultsByTarget();
       } finally { ResumeRepaint(true); }
+    }
+
+    private void relativeOrAbsoluteComboBox_SelectedIndexChanged(object sender, EventArgs e) {
+      if (suppressUpdates) return;
+      var pd = (ProblemInstance)problemComboBox.SelectedItem;
+      if (!double.IsNaN(pd.BestKnownQuality)) {
+        var max = pd.IsMaximization();
+        if (targetsAreRelative) targets = GetAbsoluteTargets(pd).ToArray();
+        else {
+          // Rounding to 5 digits since it's certainly appropriate for this application
+          if (pd.BestKnownQuality > 0) {
+            targets = targets.Select(x => Math.Round(max ? 1.0 - (x / pd.BestKnownQuality) : (x / pd.BestKnownQuality) - 1.0, 5)).ToArray();
+          } else if (pd.BestKnownQuality < 0) {
+            targets = targets.Select(x => Math.Round(!max ? 1.0 - (x / pd.BestKnownQuality) : (x / pd.BestKnownQuality) - 1.0, 5)).ToArray();
+          }
+        }
+      }
+      targetsAreRelative = (string)relativeOrAbsoluteComboBox.SelectedItem == "relative";
+      SuspendRepaint();
+      suppressTargetsEvents = true;
+      try {
+        if (targetsAreRelative)
+          targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100)) + "%";
+        else targetsTextBox.Text = string.Join(" ; ", targets);
+
+        UpdateResultsByTarget();
+      } finally { suppressTargetsEvents = false; ResumeRepaint(true); }
     }
 
@@ -823 +879 @@
             targets = dialog.Values.Select(x => (double)x).ToArray();
             suppressTargetsEvents = true;
-            targetsTextBox.Text = string.Join("% ; ", targets);
+            if (targetsAreRelative)
+              targetsTextBox.Text = string.Join("% ; ", targets);
+            else targetsTextBox.Text = string.Join(" ; ", targets);
             suppressTargetsEvents = false;
 
@@ -835 +893 @@
     private void addTargetsAsResultButton_Click(object sender, EventArgs e) {
       var table = (string)dataTableComboBox.SelectedItem;
-
-      var targetsPerProblem = CalculateBestTargetPerProblemInstance(table);
-
+      if (string.IsNullOrEmpty(table)) return;
+      
       foreach (var run in Content) {
         if (!run.Results.ContainsKey(table)) continue;
@@ -844 +901 @@
         var i = 0;
         var j = 0;
-        var pd = new ProblemDescription(run);
-        while (i < targets.Length && j < values.Count) {
-          var target = (pd.IsMaximization() ? (1 - targets[i]) : (1 + targets[i])) * targetsPerProblem[pd];
+        var pd = new ProblemInstance(run);
+        var max = pd.IsMaximization();
+        var absTargets = GetAbsoluteTargetsWorstToBest(problems.Single(x => x.Equals(pd)));
+        while (i < absTargets.Length && j < values.Count) {
+          var target = absTargets[i];
           var current = values[j];
-          if (pd.IsMaximization() && current.Item2 >= target
-              || !pd.IsMaximization() && current.Item2 <= target) {
+          if (max && current.Item2 >= target || !max && current.Item2 <= target) {
             run.Results[table + ".Target" + target] = new DoubleValue(current.Item1);
             i++;
@@ -857 +915 @@
         }
       }
+    }
+
+    private void markerCheckBox_CheckedChanged(object sender, EventArgs e) {
+      SuspendRepaint();
+      try {
+        UpdateResultsByTarget();
+      } finally { ResumeRepaint(true); }
     }
     #endregion
@@ -887 +952 @@
     }
 
-    private void eachOrAllBudgetsCheckBox_CheckedChanged(object sender, EventArgs e) {
-      var each = eachOrAllBudgetsCheckBox.Checked;
-      eachOrAllBudgetsCheckBox.Text = each ? "each" : "all";
+    private void aggregateBudgetsCheckBox_CheckedChanged(object sender, EventArgs e) {
       SuspendRepaint();
       try {
@@ -967 +1030 @@
 
     #region Helpers
-    private Dictionary<ProblemDescription, double> CalculateBestTargetPerProblemInstance(string table) {
+    private double CalculateRelativeDifference(bool maximization, double bestKnown, double fit) {
+      if (bestKnown == 0) {
+        // no relative difference with respect to bestKnown possible
+        return maximization ? -fit : fit;
+      }
+      var absDiff = (fit - bestKnown);
+      var relDiff = absDiff / bestKnown;
+      if (maximization) {
+        return bestKnown > 0 ? -relDiff : relDiff;
+      } else {
+        return bestKnown > 0 ? relDiff : -relDiff;
+      }
+    }
+
+    private Dictionary<ProblemInstance, double> CalculateBestTargetPerProblemInstance(string table) {
+      if (table == null) table = string.Empty;
       return (from r in Content
               where r.Visible
-              let pd = new ProblemDescription(r)
+              let pd = new ProblemInstance(r)
               let target = r.Parameters.ContainsKey("BestKnownQuality")
                            && r.Parameters["BestKnownQuality"] is DoubleValue
                 ? ((DoubleValue)r.Parameters["BestKnownQuality"]).Value
-                : ((IndexedDataTable<double>)r.Results[table]).Rows.First().Values.Last().Item2
+                : (r.Results.ContainsKey(table) && r.Results[table] is IndexedDataTable<double>
+                  ? ((IndexedDataTable<double>)r.Results[table]).Rows.First().Values.Last().Item2
+                  : double.NaN)
               group target by pd into g
-              select new { Problem = g.Key, Target = g.Key.IsMaximization() ? g.Max() : g.Min() })
+              select new { Problem = g.Key, Target = g.Any(x => !double.IsNaN(x)) ? (g.Key.IsMaximization() ? g.Max() : g.Where(x => !double.IsNaN(x)).Min()) : double.NaN })
         .ToDictionary(x => x.Problem, x => x.Target);
     }
@@ -990 +1070 @@
         UpdateResultsByCost();
       } finally { ResumeRepaint(true); }
+    }
+
+    private void UpdateBestKnownQualities() {
+      var table = (string)dataTableComboBox.SelectedItem;
+      if (string.IsNullOrEmpty(table)) return;
+
+      var targetsPerProblem = CalculateBestTargetPerProblemInstance(table);
+      foreach (var pd in problems) {
+        double bkq;
+        if (targetsPerProblem.TryGetValue(pd, out bkq))
+          pd.BestKnownQuality = bkq;
+        else pd.BestKnownQuality = double.NaN;
+      }
+      //problemComboBox.ResetBindings();
     }
     #endregion
@@ -1078 +1172 @@
     }
 
-    private class ProblemDescription {
+    private class ProblemInstance : INotifyPropertyChanged {
       private readonly bool matchAll;
-      public static readonly ProblemDescription MatchAll = new ProblemDescription() {
+      public static readonly ProblemInstance MatchAll = new ProblemInstance() {
         ProblemName = "All with Best-Known"
       };
 
-      private ProblemDescription() {
+      private ProblemInstance() {
         ProblemType = string.Empty;
         ProblemName = string.Empty;
@@ -1094 +1188 @@
         DisplayMaximization = false;
         matchAll = true;
-      }
-
-      public ProblemDescription(IRun run) {
+        BestKnownQuality = double.NaN;
+      }
+
+      public ProblemInstance(IRun run) {
         ProblemType = GetStringValueOrEmpty(run, "Problem Type");
         ProblemName = GetStringValueOrEmpty(run, "Problem Name");
@@ -1106 +1201 @@
         DisplayMaximization = !string.IsNullOrEmpty(Maximization);
         matchAll = false;
-      }
-
-      public bool DisplayProblemType { get; set; }
-      public string ProblemType { get; set; }
-      public bool DisplayProblemName { get; set; }
-      public string ProblemName { get; set; }
-      public bool DisplayEvaluator { get; set; }
-      public string Evaluator { get; set; }
-      public bool DisplayMaximization { get; set; }
-      public string Maximization { get; set; }
+        BestKnownQuality = GetDoubleValueOrNaN(run, "BestKnownQuality");
+      }
+
+      private bool displayProblemType;
+      public bool DisplayProblemType {
+        get { return displayProblemType; }
+        set {
+          if (displayProblemType == value) return;
+          displayProblemType = value;
+          OnPropertyChanged("DisplayProblemType");
+        }
+      }
+      private string problemType;
+      public string ProblemType {
+        get { return problemType; }
+        set {
+          if (problemType == value) return;
+          problemType = value;
+          OnPropertyChanged("ProblemType");
+        }
+      }
+      private bool displayProblemName;
+      public bool DisplayProblemName {
+        get { return displayProblemName; }
+        set {
+          if (displayProblemName == value) return;
+          displayProblemName = value;
+          OnPropertyChanged("DisplayProblemName");
+        }
+      }
+      private string problemName;
+      public string ProblemName {
+        get { return problemName; }
+        set {
+          if (problemName == value) return;
+          problemName = value;
+          OnPropertyChanged("ProblemName");
+        }
+      }
+      private bool displayEvaluator;
+      public bool DisplayEvaluator {
+        get { return displayEvaluator; }
+        set {
+          if (displayEvaluator == value) return;
+          displayEvaluator = value;
+          OnPropertyChanged("DisplayEvaluator");
+        }
+      }
+      private string evaluator;
+      public string Evaluator {
+        get { return evaluator; }
+        set {
+          if (evaluator == value) return;
+          evaluator = value;
+          OnPropertyChanged("Evaluator");
+        }
+      }
+      private bool displayMaximization;
+      public bool DisplayMaximization {
+        get { return displayMaximization; }
+        set {
+          if (displayMaximization == value) return;
+          displayMaximization = value;
+          OnPropertyChanged("DisplayMaximization");
+        }
+      }
+      private string maximization;
+      public string Maximization {
+        get { return maximization; }
+        set {
+          if (maximization == value) return;
+          maximization = value;
+          OnPropertyChanged("Maximization");
+        }
+      }
+      private double bestKnownQuality;
+      public double BestKnownQuality {
+        get { return bestKnownQuality; }
+        set {
+          if (bestKnownQuality == value) return;
+          bestKnownQuality = value;
+          OnPropertyChanged("BestKnownQuality");
+        }
+      }
 
       public bool IsMaximization() {
@@ -1129 +1298 @@
       }
 
+      private double GetDoubleValueOrNaN(IRun run, string key) {
+        IItem param;
+        if (run.Parameters.TryGetValue(key, out param)) {
+          var dv = param as DoubleValue;
+          return dv != null ? dv.Value : double.NaN;
+        }
+        return double.NaN;
+      }
+
       private string GetStringValueOrEmpty(IRun run, string key) {
-        return run.Parameters.ContainsKey(key) ? ((StringValue)run.Parameters[key]).Value : string.Empty;
+        IItem param;
+        if (run.Parameters.TryGetValue(key, out param)) {
+          var sv = param as StringValue;
+          return sv != null ? sv.Value : string.Empty;
+        }
+        return string.Empty;
       }
 
       private string GetMaximizationValueOrEmpty(IRun run, string key) {
-        return run.Parameters.ContainsKey(key) ? (((BoolValue)run.Parameters[key]).Value ? "MAX" : "MIN") : string.Empty;
+        IItem param;
+        if (run.Parameters.TryGetValue(key, out param)) {
+          var bv = param as BoolValue;
+          return bv != null ? (bv.Value ? "MAX" : "MIN") : string.Empty;
+        }
+        return string.Empty;
       }
 
       public override bool Equals(object obj) {
-        var other = obj as ProblemDescription;
+        var other = obj as ProblemInstance;
         if (other == null) return false;
         return ProblemType == other.ProblemType
@@ -1155 +1343 @@
           (DisplayProblemName ? ProblemName : string.Empty),
           (DisplayEvaluator ? Evaluator : string.Empty),
-          (DisplayMaximization ? Maximization : string.Empty)}.Where(x => !string.IsNullOrEmpty(x)));
+          (DisplayMaximization ? Maximization : string.Empty),
+          !double.IsNaN(BestKnownQuality) ? BestKnownQuality.ToString(CultureInfo.CurrentCulture.NumberFormat) : string.Empty }.Where(x => !string.IsNullOrEmpty(x)));
+      }
+
+      public event PropertyChangedEventHandler PropertyChanged;
+      protected virtual void OnPropertyChanged(string propertyName = null) {
+        var handler = PropertyChanged;
+        if (handler != null) handler(this, new PropertyChangedEventArgs(propertyName));
       }
     }