Changeset 12808 for branches/PerformanceComparison

Timestamp:

07/28/15 14:41:03 (9 years ago)

Author:

abeham

Message:

#2431:

• added ability to plot curves for multiple targets at once
• fixed bug in IndexedDataTableView regarding log-scaling
• added result that counts restarts
• fixed bugs in IRRestarter
• Set LineWidth = 2 in analyzers for chart
• fixed bugs in RLD view

Location:

branches/PerformanceComparison

Files:

• HeuristicLab.Analysis.Views/3.3/IndexedDataTableView.cs (modified) (3 diffs)
• HeuristicLab.Analysis/3.3/Optimizers/IRRestarter.cs (modified) (16 diffs)
• HeuristicLab.Analysis/3.3/QualityAnalysis/QualityPerClockAnalyzer.cs (modified) (1 diff)
• HeuristicLab.Analysis/3.3/QualityAnalysis/QualityPerEvaluationsAnalyzer.cs (modified) (1 diff)
• HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.Designer.cs (modified) (4 diffs)
• HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs (modified) (9 diffs)

branches/PerformanceComparison/HeuristicLab.Analysis.Views/3.3/IndexedDataTableView.cs

@@ -128 +128 @@
       chart.ChartAreas[0].AxisY.Title = string.Empty;
       chart.ChartAreas[0].AxisY2.Title = string.Empty;
+      chart.ChartAreas[0].AxisX.IsLogarithmic = false;
+      chart.ChartAreas[0].AxisX2.IsLogarithmic = false;
+      chart.ChartAreas[0].AxisY.IsLogarithmic = false;
+      chart.ChartAreas[0].AxisY2.IsLogarithmic = false;
       chart.Series.Clear();
       if (Content != null) {
@@ -234 +238 @@
         area.AxisX.IntervalAutoMode = IntervalAutoMode.VariableCount;
       }
-
-      area.AxisX.IsLogarithmic = Content.VisualProperties.XAxisLogScale;
-      area.AxisX2.IsLogarithmic = Content.VisualProperties.SecondXAxisLogScale;
-      area.AxisY.IsLogarithmic = Content.VisualProperties.YAxisLogScale;
-      area.AxisY2.IsLogarithmic = Content.VisualProperties.SecondYAxisLogScale;
     }
 
@@ -247 +246 @@
         a.Maximum = double.NaN;
       }
+      // chart controls handle log scaling problems not in a graceful way
+      if (chart.Series.Any(x => x.XAxisType == AxisType.Primary && (x.Points.Count == 0 || x.Points.Any(y => y.XValue <= 0))))
+        area.AxisX.IsLogarithmic = false;
+      else area.AxisX.IsLogarithmic = Content.VisualProperties.XAxisLogScale;
+      if (chart.Series.Any(x => x.XAxisType == AxisType.Secondary && (x.Points.Count == 0 || x.Points.Any(y => y.XValue <= 0))))
+        area.AxisX2.IsLogarithmic = false;
+      else area.AxisX2.IsLogarithmic = Content.VisualProperties.SecondXAxisLogScale;
+
+      if (chart.Series.Any(x => x.YAxisType == AxisType.Primary && (x.Points.Count == 0 || x.Points.Any(y => y.YValues.Min() <= 0))))
+        area.AxisY.IsLogarithmic = false;
+      else area.AxisY.IsLogarithmic = Content.VisualProperties.YAxisLogScale;
+      if (chart.Series.Any(x => x.YAxisType == AxisType.Secondary && (x.Points.Count == 0 || x.Points.Any(y => y.YValues.Min() <= 0))))
+        area.AxisY2.IsLogarithmic = false;
+      else area.AxisY2.IsLogarithmic = Content.VisualProperties.SecondYAxisLogScale;
+
       area.RecalculateAxesScale();
       area.AxisX.IsMarginVisible = false;

branches/PerformanceComparison/HeuristicLab.Analysis/3.3/Optimizers/IRRestarter.cs

@@ -33 +33 @@
 
 namespace HeuristicLab.Analysis {
-  public enum TerminationCriterium { OnlyByTime, OnlyByEvaluations, OnlyByTarget, ByTargetAndTime, ByTargetAndEvaluations, WhicheverHitsFirst, WhicheverHitsLast }
+  public enum TerminationCriterium { OnlyByTime, OnlyByEvaluations, OnlyByTarget, ByTargetAndTime, ByTargetAndEvaluations, ByTimeAndEvaluations, WhicheverHitsFirst, WhicheverHitsLast }
   /// <summary>
   /// A run in which an algorithm is executed for a certain maximum time only.
@@ -43 +43 @@
     private const string ExecutionTimeResultName = "Execution Time";
     private const string BestQualityResultName = "BestQuality";
+    private const string RandomRestartsResultName = "RandomRestarts";
 
     public string Filename { get; set; }
@@ -153 +154 @@
     }
 
-    private int lastAlgorithmEvaluatedSolutions;
-    private int lastAlgorithmEvaluatedMoves;
     [Storable]
     private double evaluations;
@@ -239 +238 @@
         return timeHit && evalHit && targetHit
           || timeHit && (TerminationCriterium == TerminationCriterium.OnlyByTime
+                      || TerminationCriterium == TerminationCriterium.WhicheverHitsFirst
                       || TerminationCriterium == TerminationCriterium.ByTargetAndTime
-                      || TerminationCriterium == TerminationCriterium.WhicheverHitsFirst)
+                      || TerminationCriterium == TerminationCriterium.ByTimeAndEvaluations)
           || evalHit && (TerminationCriterium == TerminationCriterium.OnlyByEvaluations
+                      || TerminationCriterium == TerminationCriterium.WhicheverHitsFirst
                       || TerminationCriterium == TerminationCriterium.ByTargetAndEvaluations
-                      || TerminationCriterium == TerminationCriterium.WhicheverHitsFirst)
+                      || TerminationCriterium == TerminationCriterium.ByTimeAndEvaluations)
           || targetHit && (TerminationCriterium == TerminationCriterium.OnlyByTarget
                         || TerminationCriterium == TerminationCriterium.WhicheverHitsFirst
@@ -267 +268 @@
       bestSoFar = original.bestSoFar;
       lastAlgorithmExecutionTime = original.lastAlgorithmExecutionTime;
-      lastAlgorithmEvaluatedSolutions = original.lastAlgorithmEvaluatedSolutions;
-      lastAlgorithmEvaluatedMoves = original.lastAlgorithmEvaluatedMoves;
 
       perClockAnalyzer = cloner.Clone(original.perClockAnalyzer);
@@ -294 +293 @@
       bestSoFar = double.NaN;
       lastAlgorithmExecutionTime = TimeSpan.Zero;
-      lastAlgorithmEvaluatedSolutions = 0;
-      lastAlgorithmEvaluatedMoves = 0;
 
       perClockAnalyzer = new QualityPerClockAnalyzer();
@@ -316 +313 @@
       bestSoFar = double.NaN;
       lastAlgorithmExecutionTime = TimeSpan.Zero;
-      lastAlgorithmEvaluatedSolutions = 0;
-      lastAlgorithmEvaluatedMoves = 0;
 
       perClockAnalyzer = new QualityPerClockAnalyzer();
@@ -337 +332 @@
       bestSoFar = double.NaN;
       lastAlgorithmExecutionTime = TimeSpan.Zero;
-      lastAlgorithmEvaluatedSolutions = 0;
-      lastAlgorithmEvaluatedMoves = 0;
 
       perClockAnalyzer = new QualityPerClockAnalyzer();
@@ -366 +359 @@
       BestSoFar = double.NaN;
       lastAlgorithmExecutionTime = TimeSpan.Zero;
-      lastAlgorithmEvaluatedSolutions = 0;
-      lastAlgorithmEvaluatedMoves = 0;
 
       CurrentRun = null;
@@ -399 +390 @@
         CurrentRun.Results.Add(perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName, new IntValue(0));
         CurrentRun.Results.Add(BestQualityResultName, new DoubleValue(Maximization ? double.MinValue : double.MaxValue));
+        CurrentRun.Results.Add(RandomRestartsResultName, new IntValue(0));
       }
       Algorithm.Start();
@@ -492 +484 @@
       algorithm.ExceptionOccurred += Algorithm_ExceptionOccurred;
       algorithm.ExecutionTimeChanged += Algorithm_ExecutionTimeChanged;
-      algorithm.ExecutionStateChanged += Algorithm_ExecutionStateChanged;
       algorithm.Paused += Algorithm_Paused;
       algorithm.Prepared += Algorithm_Prepared;
-      algorithm.Started += Algorithm_Started;
       algorithm.Stopped += Algorithm_Stopped;
       algorithm.ProblemChanged += Algorithm_ProblemChanged;
@@ -503 +493 @@
       algorithm.ExceptionOccurred -= Algorithm_ExceptionOccurred;
       algorithm.ExecutionTimeChanged -= Algorithm_ExecutionTimeChanged;
-      algorithm.ExecutionStateChanged -= Algorithm_ExecutionStateChanged;
       algorithm.Paused -= Algorithm_Paused;
       algorithm.Prepared -= Algorithm_Prepared;
-      algorithm.Started -= Algorithm_Started;
       algorithm.Stopped -= Algorithm_Stopped;
       algorithm.ProblemChanged -= Algorithm_ProblemChanged;
@@ -514 +502 @@
     }
     private void Algorithm_ExecutionTimeChanged(object sender, EventArgs e) {
-      if (Algorithm.ExecutionState != ExecutionState.Started) return;
-
-      if (ExecutionState == ExecutionState.Started)
-        UpdateAlgorithmResults();
-
-      if (IsFinished && ExecutionState != ExecutionState.Stopped) {
-        Algorithm.Stop();
-      }
-      OnExecutionTimeChanged();
-    }
-
-    private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
-      //OnExecutionStateChanged();
+      ExecutionTime += Algorithm.ExecutionTime - lastAlgorithmExecutionTime;
+      lastAlgorithmExecutionTime = Algorithm.ExecutionTime;
     }
     private void Algorithm_Paused(object sender, EventArgs e) {
-      UpdateAlgorithmResults();
+      ExecutionTime += Algorithm.ExecutionTime - lastAlgorithmExecutionTime;
+      lastAlgorithmExecutionTime = Algorithm.ExecutionTime;
       OnPaused();
     }
     private void Algorithm_Prepared(object sender, EventArgs e) {
-      lastAlgorithmEvaluatedSolutions = 0;
-      lastAlgorithmEvaluatedMoves = 0;
       lastAlgorithmExecutionTime = TimeSpan.Zero;
-    }
-    private void Algorithm_Started(object sender, EventArgs e) {
-      //OnStarted();
     }
     private void Algorithm_Stopped(object sender, EventArgs e) {
       var bestQuality = UpdateAlgorithmResults();
 
-      var execTime = ((TimeSpanValue)currentRun.Results[ExecutionTimeResultName]).Value;
+      var execTime = ((TimeSpanValue)CurrentRun.Results[ExecutionTimeResultName]).Value;
       foreach (var result in Algorithm.Results) {
         if (result.Name == perClockAnalyzer.QualityPerClockParameter.ResultName) {
-          if (!currentRun.Results.ContainsKey(result.Name))
-            currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
+          if (!CurrentRun.Results.ContainsKey(result.Name))
+            CurrentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
           else {
-            var dt = (IndexedDataTable<double>)currentRun.Results[result.Name];
+            var dt = (IndexedDataTable<double>)CurrentRun.Results[result.Name];
             var best = dt.Rows.First().Values.Last().Item2;
             var resultDt = (IndexedDataTable<double>)result.Value;
@@ -560 +533 @@
           }
         } else if (result.Name == perEvaluationsAnalyzer.QualityPerEvaluationsParameter.ResultName) {
-          if (!currentRun.Results.ContainsKey(result.Name))
-            currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
+          if (!CurrentRun.Results.ContainsKey(result.Name))
+            CurrentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
           else {
-            var dt = (IndexedDataTable<double>)currentRun.Results[result.Name];
-            var evalSols = ((IntValue)currentRun.Results[perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName]).Value;
-            var evalMoves = ((IntValue)currentRun.Results[perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName]).Value;
+            var dt = (IndexedDataTable<double>)CurrentRun.Results[result.Name];
+            var evalSols = ((IntValue)CurrentRun.Results[perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName]).Value;
+            var evalMoves = ((IntValue)CurrentRun.Results[perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName]).Value;
             var best = dt.Rows.First().Values.Last().Item2;
             var resultDt = (IndexedDataTable<double>)result.Value;
@@ -576 +549 @@
           }
         } else if (result.Name == perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName) {
-          var evalSols = ((IntValue)currentRun.Results[perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName]);
-          evalSols.Value += ((IntValue)result.Value).Value;
+          var evalSols = ((IntValue)CurrentRun.Results[perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName]).Value;
+          CurrentRun.Results[perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName] = new IntValue(evalSols + ((IntValue)result.Value).Value);
         } else if (result.Name == perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName) {
-          var evalMoves = ((IntValue)currentRun.Results[perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName]);
-          evalMoves.Value += ((IntValue)result.Value).Value;
+          var evalMoves = ((IntValue)CurrentRun.Results[perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName]).Value;
+          CurrentRun.Results[perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName] = new IntValue(evalMoves + ((IntValue)result.Value).Value);
         } else if (result.Name == perEvaluationsAnalyzer.BestQualityParameter.ActualName) {
-          var best = ((DoubleValue)currentRun.Results[BestQualityResultName]).Value;
+          var best = ((DoubleValue)CurrentRun.Results[BestQualityResultName]).Value;
           if (Maximization && best < bestQuality || !Maximization && best > bestQuality)
-            currentRun.Results[BestQualityResultName] = new DoubleValue(bestQuality);
+            CurrentRun.Results[BestQualityResultName] = new DoubleValue(bestQuality);
         } else if (result.Name.ToLower().EndsWith("solution") && BestSoFar == bestQuality) {
-          if (currentRun.Results.ContainsKey(result.Name))
-            currentRun.Results[result.Name] = (IItem)result.Value.Clone();
-          else currentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
+          if (CurrentRun.Results.ContainsKey(result.Name))
+            CurrentRun.Results[result.Name] = (IItem)result.Value.Clone();
+          else CurrentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
         }
       }
-      currentRun.Results[ExecutionTimeResultName] = new TimeSpanValue(execTime + Algorithm.ExecutionTime);
+      CurrentRun.Results[ExecutionTimeResultName] = new TimeSpanValue(execTime + Algorithm.ExecutionTime);
 
       if (!forceStop && !IsFinished) {
+        CurrentRun.Results[RandomRestartsResultName] = new IntValue(1 + ((IntValue)CurrentRun.Results[RandomRestartsResultName]).Value);
         Algorithm.Prepare();
         Algorithm.Start();
       } else {
         forceStop = false;
-        Runs.Add(currentRun);
-        currentRun = null;
+        Runs.Add(CurrentRun);
         Algorithm.Prepare(true);
         ExecutionState = ExecutionState.Stopped;
@@ -613 +586 @@
       if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName, out evaluationsResult)) {
         var evals = ((IntValue)evaluationsResult.Value).Value;
-        Evaluations += evals - lastAlgorithmEvaluatedSolutions;
-        lastAlgorithmEvaluatedSolutions = evals;
+        Evaluations += evals;
       }
       if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName, out evaluationsResult)) {
         var evals = ((IntValue)evaluationsResult.Value).Value;
-        Evaluations += moveCostPerSolution * (evals - lastAlgorithmEvaluatedMoves);
-        lastAlgorithmEvaluatedMoves = evals;
+        Evaluations += moveCostPerSolution * evals;
       }
       if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.BestQualityParameter.ActualName, out evaluationsResult)) {

branches/PerformanceComparison/HeuristicLab.Analysis/3.3/QualityAnalysis/QualityPerClockAnalyzer.cs

@@ -65 +65 @@
         Rows = { new IndexedDataRow<double>("First-hit Graph") { VisualProperties = {
           ChartType = DataRowVisualProperties.DataRowChartType.StepLine,
-          LineWidth = 3
+          LineWidth = 2
         } } }
       };

branches/PerformanceComparison/HeuristicLab.Analysis/3.3/QualityAnalysis/QualityPerEvaluationsAnalyzer.cs

@@ -71 +71 @@
         Rows = { new IndexedDataRow<double>("First-hit Graph") { VisualProperties = {
           ChartType = DataRowVisualProperties.DataRowChartType.StepLine,
-          LineWidth = 3
+          LineWidth = 2
         } } }
       };

branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.Designer.cs

@@ -60 +60 @@
       this.budgetsTextBox = new System.Windows.Forms.TextBox();
       this.fixedCostButton = new System.Windows.Forms.Button();
+      this.allOrEachTargetCheckBox = new System.Windows.Forms.CheckBox();
       ((System.ComponentModel.ISupportInitialize)(this.errorProvider)).BeginInit();
       this.SuspendLayout();
@@ -139 +140 @@
       this.targetsTextBox.Location = new System.Drawing.Point(69, 57);
       this.targetsTextBox.Name = "targetsTextBox";
-      this.targetsTextBox.Size = new System.Drawing.Size(265, 20);
+      this.targetsTextBox.Size = new System.Drawing.Size(209, 20);
       this.targetsTextBox.TabIndex = 6;
       this.toolTip.SetToolTip(this.targetsTextBox, "The order of the targets is important, first to-hit targets\r\nshould be given firs" +
@@ -213 +214 @@
       this.fixedCostButton.Click += new System.EventHandler(this.fixedCostButton_Click);
       // 
+      // allOrEachTargetCheckBox
+      // 
+      this.allOrEachTargetCheckBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
+      this.allOrEachTargetCheckBox.AutoSize = true;
+      this.allOrEachTargetCheckBox.Location = new System.Drawing.Point(298, 59);
+      this.allOrEachTargetCheckBox.Name = "allOrEachTargetCheckBox";
+      this.allOrEachTargetCheckBox.Size = new System.Drawing.Size(36, 17);
+      this.allOrEachTargetCheckBox.TabIndex = 8;
+      this.allOrEachTargetCheckBox.Text = "all";
+      this.allOrEachTargetCheckBox.UseVisualStyleBackColor = true;
+      this.allOrEachTargetCheckBox.CheckedChanged += new System.EventHandler(this.allOrEachTargetCheckBox_CheckedChanged);
+      // 
       // RunCollectionRLDView
       // 
       this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Inherit;
+      this.Controls.Add(this.allOrEachTargetCheckBox);
       this.Controls.Add(this.generateTargetsButton);
       this.Controls.Add(this.fixedCostButton);
@@ -254 +268 @@
     private System.Windows.Forms.TextBox budgetsTextBox;
     private System.Windows.Forms.Label budgetsLabel;
+    private System.Windows.Forms.CheckBox allOrEachTargetCheckBox;
   }
 }

branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs

@@ -27 +27 @@
 using HeuristicLab.Analysis;
 using HeuristicLab.Collections;
-using HeuristicLab.Common;
 using HeuristicLab.Core.Views;
 using HeuristicLab.Data;
@@ -55 +54 @@
       InitializeComponent();
       combinedDataTable = new IndexedDataTable<double>("Combined DataTable", "A data table containing the ECDF of each of a number of groups.") {
-        VisualProperties = { YAxisTitle = "Proportion of reached targets" }
+        VisualProperties = {
+          YAxisTitle = "Proportion of reached targets",
+          YAxisMinimumFixedValue = 0, YAxisMinimumAuto = false,
+          YAxisMaximumFixedValue = 1, YAxisMaximumAuto = false
+        }
       };
       viewHost.Content = combinedDataTable;
@@ -179 +182 @@
         if (string.IsNullOrEmpty(table)) return;
         if (levels == null) {
-          var worst = Content.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Max(y => y.Item2)).First();
-          var best = Content.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Min(y => y.Item2)).Last();
+          var worstMin = Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.First().Item2).Min();
+          var worstMax = Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.First().Item2).Max();
+          var bestMin = Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Last().Item2).Min();
+          var bestMax = Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Last().Item2).Max();
+          double worst, best;
+          if (Math.Abs(bestMax - worstMin) > Math.Abs(bestMin - worstMax)) {
+            worst = worstMin;
+            best = bestMax;
+          } else {
+            worst = worstMax;
+            best = bestMin;
+          }
           levels = Enumerable.Range(0, 11).Select(x => worst + (x / 10.0) * (best - worst)).ToArray();
           suppressTargetsEvents = true;
@@ -196 +209 @@
         var xAxisTitles = new HashSet<string>();
         foreach (var group in groupedRuns) {
-          var hits = new SortedList<double, int>();
+          var hits = new Dictionary<string, SortedList<double, double>>();
           foreach (var run in group.Item2) {
+            if (!run.Results.ContainsKey(table)) continue;
             var resultsTable = (IndexedDataTable<double>)run.Results[table];
             xAxisTitles.Add(resultsTable.VisualProperties.XAxisTitle);
             var values = resultsTable.Rows.First().Values;
             var maximization = values.First().Item2 < values.Last().Item2;
-            var i = 0;
-            var j = 0;
-            var current = values[j];
-            var prev = Tuple.Create(-1.0, double.NaN);
-            while (i < levels.Length) {
-              if ((double.IsNaN(prev.Item2) || prev.Item2 != current.Item2)
-                  && (maximization && current.Item2 >= levels[i]
-                  || !maximization && current.Item2 <= levels[i])) {
-                if (hits.ContainsKey(current.Item1))
-                  hits[current.Item1]++;
-                else hits[current.Item1] = 1;
-                i++;
-              } else {
-                j++;
-                if (j >= values.Count) break;
-                prev = current;
-                current = values[j];
+            if (allOrEachTargetCheckBox.Checked) {
+              // each
+              foreach (double l in levels) {
+                var key = "-" + l.ToString();
+                if (!hits.ContainsKey(key)) hits.Add(key, new SortedList<double, double>());
+                foreach (var v in values) {
+                  if (maximization && v.Item2 >= l || !maximization && v.Item2 <= l) {
+                    if (hits[key].ContainsKey(v.Item1))
+                      hits[key][v.Item1] += 1.0 / group.Item2.Count;
+                    else hits[key][v.Item1] = 1.0 / group.Item2.Count;
+                    break;
+                  }
+                }
+              }
+            } else {
+              if (!hits.ContainsKey("all")) hits.Add("all", new SortedList<double, double>());
+              // all
+              var i = 0;
+              var j = 0;
+              var current = values[j];
+              var prev = Tuple.Create(-1.0, double.NaN);
+              while (i < levels.Length) {
+                if ((double.IsNaN(prev.Item2) || prev.Item2 != current.Item2)
+                    && (maximization && current.Item2 >= levels[i]
+                        || !maximization && current.Item2 <= levels[i])) {
+                  if (hits["all"].ContainsKey(current.Item1))
+                    hits["all"][current.Item1] += 1.0 / (group.Item2.Count * levels.Length);
+                  else hits["all"][current.Item1] = 1.0 / (group.Item2.Count * levels.Length);
+                  i++;
+                } else {
+                  j++;
+                  if (j >= values.Count) break;
+                  prev = current;
+                  current = values[j];
+                }
               }
             }
           }
-          var row = new IndexedDataRow<double>(group.Item1) { VisualProperties = { ChartType = DataRowVisualProperties.DataRowChartType.StepLine } };
-          var total = 0.0;
-          foreach (var h in hits) {
-            total += h.Value;
-            row.Values.Add(Tuple.Create(h.Key, total / (group.Item2.Count * levels.Length)));
-          }
-          combinedDataTable.Rows.Add(row);
+          foreach (var list in hits) {
+            var row = new IndexedDataRow<double>(group.Item1 + (list.Key != "all" ? list.Key : string.Empty)) {
+              VisualProperties = {
+                ChartType = DataRowVisualProperties.DataRowChartType.StepLine,
+                LineWidth = 2
+              }
+            };
+            var total = 0.0;
+            foreach (var h in list.Value) {
+              total += h.Value;
+              row.Values.Add(Tuple.Create(h.Key, total));
+            }
+            combinedDataTable.Rows.Add(row);
+          }
         }
         combinedDataTable.VisualProperties.XAxisTitle = string.Join(" / ", xAxisTitles);
-        combinedDataTable.VisualProperties.XAxisLogScale = logScalingCheckBox.Checked;
+        combinedDataTable.VisualProperties.XAxisLogScale = combinedDataTable.Rows.Count > 0 && logScalingCheckBox.Checked;
       } finally { ResumeRepaint(true); }
     }
@@ -317 +356 @@
       } else if (Content.Count > 0 && dataTableComboBox.SelectedIndex >= 0) {
         var table = (string)dataTableComboBox.SelectedItem;
-        var worst = Content.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Max(y => y.Item2)).First();
-        var best = Content.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Min(y => y.Item2)).Last();
+        var worst = Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Max(y => y.Item2)).First();
+        var best = Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Min(y => y.Item2)).Last();
         max = (decimal)Math.Max(best, worst);
         min = (decimal)Math.Min(best, worst);
@@ -332 +371 @@
             levels = maximization ? dialog.Values.Select(x => (double)x).ToArray()
                                   : dialog.Values.Reverse().Select(x => (double)x).ToArray();
+            suppressTargetsEvents = true;
+            targetsTextBox.Text = string.Join(" ; ", levels);
+            suppressTargetsEvents = false;
+
             UpdateRuns();
             SetEnabledStateOfControls();
@@ -342 +385 @@
       var table = (string)dataTableComboBox.SelectedItem;
       foreach (var run in Content) {
+        if (!run.Results.ContainsKey(table)) continue;
         var resultsTable = (IndexedDataTable<double>)run.Results[table];
         var values = resultsTable.Rows.First().Values;
@@ -384 +428 @@
 
       foreach (var run in Content) {
+        if (!run.Results.ContainsKey(table)) continue;
         var resultsTable = (IndexedDataTable<double>)run.Results[table];
         var values = resultsTable.Rows.First().Values;
@@ -403 +448 @@
       }
     }
+
+    private void allOrEachTargetCheckBox_CheckedChanged(object sender, EventArgs e) {
+      var each = allOrEachTargetCheckBox.Checked;
+      allOrEachTargetCheckBox.Text = each ? "each" : "all";
+      UpdateRuns();
+    }
   }
 }