source: stable/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs @ 16836

Last change on this file since 16836 was 16836, checked in by gkronber, 15 months ago

#3000 merged r16716 and r16795 from trunk to stable

File size: 63.6 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections;
24using System.Collections.Generic;
25using System.ComponentModel;
26using System.Drawing;
27using System.Globalization;
28using System.Linq;
29using System.Windows.Forms;
30using System.Windows.Forms.DataVisualization.Charting;
31using HeuristicLab.Analysis;
32using HeuristicLab.Collections;
33using HeuristicLab.Common;
34using HeuristicLab.Core;
35using HeuristicLab.Core.Views;
36using HeuristicLab.Data;
37using HeuristicLab.MainForm;
38using HeuristicLab.MainForm.WindowsForms;
39
40namespace HeuristicLab.Optimization.Views {
41  [View("Run-length Distribution View")]
42  [Content(typeof(RunCollection), false)]
43  public partial class RunCollectionRLDView : ItemView {
44    private List<Series> invisibleTargetSeries;
45
46    private const string AllInstances = "All Instances";
47
48    private static readonly Color[] colors = new[] {
49      Color.FromArgb(0x40, 0x6A, 0xB7),
50      Color.FromArgb(0xB1, 0x6D, 0x01),
51      Color.FromArgb(0x4E, 0x8A, 0x06),
52      Color.FromArgb(0x75, 0x50, 0x7B),
53      Color.FromArgb(0x72, 0x9F, 0xCF),
54      Color.FromArgb(0xA4, 0x00, 0x00),
55      Color.FromArgb(0xAD, 0x7F, 0xA8),
56      Color.FromArgb(0x29, 0x50, 0xCF),
57      Color.FromArgb(0x90, 0xB0, 0x60),
58      Color.FromArgb(0xF5, 0x89, 0x30),
59      Color.FromArgb(0x55, 0x57, 0x53),
60      Color.FromArgb(0xEF, 0x59, 0x59),
61      Color.FromArgb(0xED, 0xD4, 0x30),
62      Color.FromArgb(0x63, 0xC2, 0x16),
63    };
64    private static readonly ChartDashStyle[] lineStyles = new[] {
65      ChartDashStyle.Solid,
66      ChartDashStyle.Dash,
67      ChartDashStyle.DashDot,
68      ChartDashStyle.Dot
69    };
70    private static readonly DataRowVisualProperties.DataRowLineStyle[] hlLineStyles = new[] {
71      DataRowVisualProperties.DataRowLineStyle.Solid,
72      DataRowVisualProperties.DataRowLineStyle.Dash,
73      DataRowVisualProperties.DataRowLineStyle.DashDot,
74      DataRowVisualProperties.DataRowLineStyle.Dot
75    };
76
77    public new RunCollection Content {
78      get { return (RunCollection)base.Content; }
79      set { base.Content = value; }
80    }
81
82    private List<AlgorithmInstance> groups;
83
84    private double[] targets;
85    private double[] budgets;
86    private bool targetsAreRelative = true;
87    private bool showLabelsInTargetChart = true;
88    private readonly BindingList<ProblemInstance> problems;
89
90    private bool updateInProgress;
91    private bool suppressContentEvents;
92    private readonly IndexedDataTable<double> byCostDataTable;
93    public IndexedDataTable<double> ByCostDataTable {
94      get { return byCostDataTable; }
95    }
96
97    public RunCollectionRLDView() {
98      InitializeComponent();
99      invisibleTargetSeries = new List<Series>();
100
101      try {
102        updateInProgress = true;
103        targetChart.CustomizeAllChartAreas();
104        targetChart.ChartAreas[0].CursorX.Interval = 1;
105        targetChart.SuppressExceptions = true;
106        byCostDataTable = new IndexedDataTable<double>("ECDF by Cost", "A data table containing the ECDF of function values (relative to best-known).") {
107          VisualProperties = {
108            YAxisTitle = "Proportion of runs",
109            YAxisMinimumFixedValue = 0,
110            YAxisMinimumAuto = false,
111            YAxisMaximumFixedValue = 1,
112            YAxisMaximumAuto = false
113          }
114        };
115        byCostViewHost.Content = byCostDataTable;
116
117        relativeOrAbsoluteComboBox.SelectedItem = targetsAreRelative ? "relative" : "absolute";
118        problems = new BindingList<ProblemInstance>();
119        problemComboBox.DataSource = new BindingSource() { DataSource = problems };
120        problemComboBox.DataBindings.DefaultDataSourceUpdateMode = DataSourceUpdateMode.OnPropertyChanged;
121      } finally { updateInProgress = false; }
122    }
123
124    #region Content events
125    protected override void RegisterContentEvents() {
126      base.RegisterContentEvents();
127      Content.ItemsAdded += Content_ItemsAdded;
128      Content.ItemsRemoved += Content_ItemsRemoved;
129      Content.CollectionReset += Content_CollectionReset;
130      Content.UpdateOfRunsInProgressChanged += Content_UpdateOfRunsInProgressChanged;
131      Content.OptimizerNameChanged += Content_AlgorithmNameChanged;
132    }
133    protected override void DeregisterContentEvents() {
134      Content.ItemsAdded -= Content_ItemsAdded;
135      Content.ItemsRemoved -= Content_ItemsRemoved;
136      Content.CollectionReset -= Content_CollectionReset;
137      Content.UpdateOfRunsInProgressChanged -= Content_UpdateOfRunsInProgressChanged;
138      Content.OptimizerNameChanged -= Content_AlgorithmNameChanged;
139      base.DeregisterContentEvents();
140    }
141
142    private void Content_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IRun> e) {
143      foreach (var run in e.Items) RegisterRunEvents(run);
144      if (suppressContentEvents) return;
145      if (InvokeRequired) {
146        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_ItemsAdded), sender, e);
147        return;
148      }
149      if (updateInProgress) return;
150      try {
151        updateInProgress = true;
152        UpdateComboBoxes();
153        GroupRuns();
154      } finally { updateInProgress = false; }
155    }
156    private void Content_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IRun> e) {
157      foreach (var run in e.Items) DeregisterRunEvents(run);
158      if (suppressContentEvents) return;
159      if (InvokeRequired) {
160        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_ItemsRemoved), sender, e);
161        return;
162      }
163      if (updateInProgress) return;
164      try {
165        updateInProgress = true;
166        UpdateComboBoxes();
167        GroupRuns();
168      } finally { updateInProgress = false; }
169    }
170    private void Content_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
171      foreach (var run in e.OldItems) DeregisterRunEvents(run);
172      foreach (var run in e.Items) RegisterRunEvents(run);
173      if (suppressContentEvents) return;
174      if (InvokeRequired) {
175        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset), sender, e);
176        return;
177      }
178      if (updateInProgress) return;
179      try {
180        updateInProgress = true;
181        UpdateComboBoxes();
182        GroupRuns();
183      } finally { updateInProgress = false; }
184    }
185    private void Content_AlgorithmNameChanged(object sender, EventArgs e) {
186      if (InvokeRequired)
187        Invoke(new EventHandler(Content_AlgorithmNameChanged), sender, e);
188      else UpdateCaption();
189    }
190    private void Content_UpdateOfRunsInProgressChanged(object sender, EventArgs e) {
191      if (InvokeRequired) {
192        Invoke(new EventHandler(Content_UpdateOfRunsInProgressChanged), sender, e);
193        return;
194      }
195      suppressContentEvents = Content.UpdateOfRunsInProgress;
196      if (!suppressContentEvents) {
197        if (updateInProgress) return;
198        try {
199          updateInProgress = true;
200          UpdateComboBoxes();
201          GroupRuns();
202        } finally { updateInProgress = false; }
203      }
204    }
205
206    private void RegisterRunEvents(IRun run) {
207      run.PropertyChanged += run_PropertyChanged;
208    }
209    private void DeregisterRunEvents(IRun run) {
210      run.PropertyChanged -= run_PropertyChanged;
211    }
212    private void run_PropertyChanged(object sender, PropertyChangedEventArgs e) {
213      if (suppressContentEvents) return;
214      if (InvokeRequired) {
215        Invoke((Action<object, PropertyChangedEventArgs>)run_PropertyChanged, sender, e);
216      } else {
217        if (e.PropertyName == "Visible") {
218          if (updateInProgress) return;
219          try {
220            updateInProgress = true;
221            UpdateRuns();
222          } finally { updateInProgress = false; }
223        }
224      }
225    }
226    #endregion
227
228    protected override void OnContentChanged() {
229      base.OnContentChanged();
230      dataTableComboBox.Items.Clear();
231      groupComboBox.Items.Clear();
232      targetChart.ChartAreas[0].AxisX.IsLogarithmic = false;
233      targetChart.Series.Clear();
234      invisibleTargetSeries.Clear();
235      byCostDataTable.VisualProperties.XAxisLogScale = false;
236      byCostDataTable.Rows.Clear();
237
238      UpdateCaption();
239      if (Content != null) {
240        try {
241          updateInProgress = true;
242          UpdateComboBoxes();
243          UpdateRuns();
244        } finally { updateInProgress = false; }
245      }
246    }
247
248
249    private void UpdateComboBoxes() {
250      var selectedGroupItem = (string)groupComboBox.SelectedItem;
251
252      var groupings = Content.ParameterNames.OrderBy(x => x).ToArray();
253      groupComboBox.Items.Clear();
254      groupComboBox.Items.Add(AllInstances);
255      groupComboBox.Items.AddRange(groupings);
256      if (selectedGroupItem != null && groupComboBox.Items.Contains(selectedGroupItem)) {
257        groupComboBox.SelectedItem = selectedGroupItem;
258      } else if (groupComboBox.Items.Count > 0) {
259        groupComboBox.SelectedItem = groupComboBox.Items[0];
260      }
261
262      string selectedDataTable = (string)dataTableComboBox.SelectedItem;
263
264      dataTableComboBox.Items.Clear();
265      var dataTables = (from run in Content
266                        from result in run.Results
267                        where result.Value is IndexedDataTable<double>
268                        select result.Key).Distinct().ToArray();
269
270      dataTableComboBox.Items.AddRange(dataTables);
271      if (selectedDataTable != null && dataTableComboBox.Items.Contains(selectedDataTable)) {
272        dataTableComboBox.SelectedItem = selectedDataTable;
273      } else if (dataTableComboBox.Items.Count > 0) {
274        dataTableComboBox.SelectedItem = dataTableComboBox.Items[0];
275      }
276
277      var selectedProblemItem = (ProblemInstance)problemComboBox.SelectedItem;
278     
279      UpdateProblemInstances();
280     
281      foreach (var p in problems) {
282        if (p.Equals(selectedProblemItem))
283          problemComboBox.SelectedItem = p;
284      }
285
286      if (selectedProblemItem == null && problems.Count > 1) problemComboBox.SelectedItem = problems[1];
287
288      SetEnabledStateOfControls();
289    }
290
291    protected override void SetEnabledStateOfControls() {
292      base.SetEnabledStateOfControls();
293      groupComboBox.Enabled = Content != null;
294      problemComboBox.Enabled = Content != null && problemComboBox.Items.Count > 1;
295      dataTableComboBox.Enabled = Content != null && dataTableComboBox.Items.Count > 1;
296      addTargetsAsResultButton.Enabled = Content != null && targets != null && dataTableComboBox.SelectedIndex >= 0;
297      addBudgetsAsResultButton.Enabled = Content != null && budgets != null && dataTableComboBox.SelectedIndex >= 0;
298      generateTargetsButton.Enabled = targets != null;
299    }
300
301    private void GroupRuns() {
302      groups = new List<AlgorithmInstance>();
303
304      var table = (string)dataTableComboBox.SelectedItem;
305      if (string.IsNullOrEmpty(table)) return;
306
307      var selectedGroup = (string)groupComboBox.SelectedItem;
308      if (string.IsNullOrEmpty(selectedGroup)) return;
309
310      var selectedProblem = (ProblemInstance)problemComboBox.SelectedItem;
311      if (selectedProblem == null) return;
312
313      foreach (var alg in from r in Content
314                          where (selectedGroup == AllInstances || r.Parameters.ContainsKey(selectedGroup))
315                            && r.Visible
316                            && selectedProblem.Match(r)
317                          let key = selectedGroup == AllInstances
318                            ? AllInstances : r.Parameters[selectedGroup].ToString()
319                          group r by key into g
320                          select g) {
321        var trials = (from run in alg
322                     from pd in problems.Skip(1) // exclude artificial match all
323                     where pd.Match(run) && run.Results.ContainsKey(table)
324                     let cgraph = run.Results[table] as IndexedDataTable<double>
325                     where cgraph != null && cgraph.Rows.Count > 0
326                        && cgraph.Rows.First().Values.Count > 0
327                     group cgraph by pd into g
328                     select g).ToList();
329
330        if (trials.Count == 0) continue;
331        groups.Add(new AlgorithmInstance(alg.Key, trials));
332      }
333    }
334
335    private void UpdateProblemInstances() {
336      try {
337        problems.Clear();
338        var table = (string)dataTableComboBox.SelectedItem;
339       
340        var problemDict = CalculateBestTargetPerProblemInstance(table);
341
342        var problemTypesDifferent = problemDict.Keys.Select(x => x.ProblemType).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
343        var problemNamesDifferent = problemDict.Keys.Select(x => x.ProblemName).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
344        var evaluatorDifferent = problemDict.Keys.Select(x => x.Evaluator).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
345        var maximizationDifferent = problemDict.Keys.Select(x => x.Maximization).Distinct().Count() > 1;
346        var allEqual = !problemTypesDifferent && !problemNamesDifferent && !evaluatorDifferent && !maximizationDifferent;
347
348        problems.Add(ProblemInstance.MatchAll);
349        foreach (var p in problemDict.OrderBy(x => x.Key.ProblemName, new NaturalStringComparer()).ToList()) {
350          p.Key.BestKnownQuality = p.Value;
351          p.Key.DisplayProblemType = problemTypesDifferent;
352          p.Key.DisplayProblemName = problemNamesDifferent || allEqual;
353          p.Key.DisplayEvaluator = evaluatorDifferent;
354          p.Key.DisplayMaximization = maximizationDifferent;
355          problems.Add(p.Key);
356        }
357      } finally { ((BindingSource)problemComboBox.DataSource).ResetBindings(false); }
358    }
359
360    #region Performance analysis by (multiple) target(s)
361    private void UpdateResultsByTarget() {
362      // necessary to reset log scale -> empty chart cannot use log scaling
363      targetChart.ChartAreas[0].AxisX.IsLogarithmic = false;
364      targetChart.Series.Clear();
365      invisibleTargetSeries.Clear();
366     
367      var table = (string)dataTableComboBox.SelectedItem;
368      if (string.IsNullOrEmpty(table)) return;
369
370      if (targets == null) GenerateDefaultTargets();
371     
372      if (groups.Count == 0) return;
373
374      var xAxisTitles = new HashSet<string>();
375
376      // hits describes the number of target hits at a certain time for a certain group
377      var hits = new Dictionary<string, SortedList<double, int>>();
378      // misses describes the number of target misses after a certain time for a certain group
379      // for instance when a run ends, but has not achieved all targets, misses describes
380      // how many targets have been left open at the point when the run ended
381      var misses = new Dictionary<string, SortedList<double, int>>();
382      var totalRuns = new Dictionary<string, int>();
383
384      var aggregate = aggregateTargetsCheckBox.Checked;
385      double minEff = double.MaxValue, maxEff = double.MinValue;
386      foreach (var alg in groups) {
387        var noRuns = 0;
388        SortedList<double, int> epdfHits = null, epdfMisses = null;
389        if (aggregate) {
390          hits[alg.Name] = epdfHits = new SortedList<double, int>();
391          misses[alg.Name] = epdfMisses = new SortedList<double, int>();
392        }
393        foreach (var problem in alg.GetCases().Where(x => x.Maximization.HasValue)) {
394          var max = problem.Maximization.Value;
395          var absTargets = GetAbsoluteTargets(problem).ToArray();
396          foreach (var run in alg.GetTrials(problem)) {
397            noRuns++;
398            xAxisTitles.Add(run.XAxisName);
399
400            var efforts = absTargets.Select(t => GetEffortToHitTarget(run, t, max)).ToArray();
401            minEff = Math.Min(minEff, efforts.Min(x => x.Item2));
402            maxEff = Math.Max(maxEff, efforts.Max(x => x.Item2));
403            for (var idx = 0; idx < efforts.Length; idx++) {
404              var e = efforts[idx];
405              if (!aggregate) {
406                var key = alg.Name + "@" + (targetsAreRelative
407                            ? (targets[idx] * 100).ToString(CultureInfo.CurrentCulture.NumberFormat) + "%"
408                            : targets[idx].ToString(CultureInfo.CurrentCulture.NumberFormat));
409                if (!hits.TryGetValue(key, out epdfHits))
410                  hits[key] = epdfHits = new SortedList<double, int>();
411                if (!misses.TryGetValue(key, out epdfMisses))
412                  misses[key] = epdfMisses = new SortedList<double, int>();
413                totalRuns[key] = noRuns;
414              };
415              var list = e.Item1 ? epdfHits : epdfMisses;
416              int v;
417              if (list.TryGetValue(e.Item2, out v))
418                list[e.Item2] = v + 1;
419              else list[e.Item2] = 1;
420            }
421          }
422        }
423        if (aggregate) totalRuns[alg.Name] = noRuns;
424      }
425
426      UpdateTargetChartAxisXBounds(minEff, maxEff);
427
428      DrawTargetsEcdf(hits, misses, totalRuns);
429
430      if (targets.Length == 1) {
431        if (targetsAreRelative)
432          targetChart.ChartAreas[0].AxisY.Title = "Probability to be " + (targets[0] * 100) + "% worse than best";
433        else targetChart.ChartAreas[0].AxisY.Title = "Probability to reach at least a fitness of " + targets[0];
434      } else targetChart.ChartAreas[0].AxisY.Title = "Proportion of reached targets";
435      targetChart.ChartAreas[0].AxisX.Title = string.Join(" / ", xAxisTitles);
436      targetChart.ChartAreas[0].AxisX.IsLogarithmic = CanDisplayLogarithmic();
437      targetChart.ChartAreas[0].CursorY.Interval = 0.05;
438
439      UpdateErtTables();
440    }
441
442    private void DrawTargetsEcdf(Dictionary<string, SortedList<double, int>> hits, Dictionary<string, SortedList<double, int>> misses, Dictionary<string, int> noRuns) {
443      var colorCount = 0;
444      var lineStyleCount = 0;
445     
446      var showMarkers = markerCheckBox.Checked;
447      foreach (var list in hits) {
448        var row = new Series(list.Key) {
449          ChartType = list.Value.Count > 1000 ? SeriesChartType.FastLine : SeriesChartType.StepLine,
450          BorderWidth = 3,
451          Color = colors[colorCount],
452          BorderDashStyle = lineStyles[lineStyleCount],
453        };
454        var rowShade = new Series(list.Key + "-range") {
455          IsVisibleInLegend = false,
456          ChartType = SeriesChartType.Range,
457          Color = Color.FromArgb(32, colors[colorCount]),
458          YValuesPerPoint = 2
459        };
460
461        var ecdf = 0.0;
462        var missedecdf = 0.0;
463        var iter = misses[list.Key].GetEnumerator();
464        var moreMisses = iter.MoveNext();
465        var totalTargets = noRuns[list.Key];
466        if (aggregateTargetsCheckBox.Checked) totalTargets *= targets.Length;
467        var movingTargets = totalTargets;
468        var labelPrinted = false;
469        foreach (var h in list.Value) {
470          var prevmissedecdf = missedecdf;
471          while (moreMisses && iter.Current.Key < h.Key) {
472            if (!labelPrinted && row.Points.Count > 0) {
473              var point = row.Points.Last();
474              if (showLabelsInTargetChart)
475                point.Label = row.Name;
476              point.MarkerStyle = MarkerStyle.Cross;
477              point.MarkerBorderWidth = 1;
478              point.MarkerSize = 10;
479              labelPrinted = true;
480              rowShade.Points.Add(new DataPoint(point.XValue, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
481            }
482            missedecdf += iter.Current.Value;
483            movingTargets -= iter.Current.Value;
484            if (row.Points.Count > 0 && row.Points.Last().XValue == iter.Current.Key) {
485              row.Points.Last().SetValueY(ecdf / movingTargets);
486            } else {
487              var dp = new DataPoint(iter.Current.Key, ecdf / movingTargets);
488              if (showMarkers) {
489                dp.MarkerStyle = MarkerStyle.Circle;
490                dp.MarkerBorderWidth = 1;
491                dp.MarkerSize = 5;
492              }
493              row.Points.Add(dp);
494              prevmissedecdf = missedecdf;
495            }
496            if (boundShadingCheckBox.Checked) {
497              if (rowShade.Points.Count > 0 && rowShade.Points.Last().XValue == iter.Current.Key)
498                rowShade.Points.Last().SetValueY(ecdf / totalTargets, (ecdf + missedecdf) / totalTargets);
499              else rowShade.Points.Add(new DataPoint(iter.Current.Key, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
500            }
501            moreMisses = iter.MoveNext();
502            if (!labelPrinted) {
503              var point = row.Points.Last();
504              if (showLabelsInTargetChart)
505                point.Label = row.Name;
506              point.MarkerStyle = MarkerStyle.Cross;
507              point.MarkerBorderWidth = 1;
508              point.MarkerSize = 10;
509              labelPrinted = true;
510            }
511          }
512          ecdf += h.Value;
513          if (row.Points.Count > 0 && row.Points.Last().XValue == h.Key) {
514            row.Points.Last().SetValueY(ecdf / movingTargets);
515          } else {
516            var dp = new DataPoint(h.Key, ecdf / movingTargets);
517            if (showMarkers) {
518              dp.MarkerStyle = MarkerStyle.Circle;
519              dp.MarkerBorderWidth = 1;
520              dp.MarkerSize = 5;
521            }
522            row.Points.Add(dp);
523          }
524          if (missedecdf > 0 && boundShadingCheckBox.Checked) {
525            if (rowShade.Points.Count > 0 && rowShade.Points.Last().XValue == h.Key)
526              rowShade.Points.Last().SetValueY(ecdf / totalTargets, (ecdf + missedecdf) / totalTargets);
527            else rowShade.Points.Add(new DataPoint(h.Key, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
528          }
529        }
530
531        while (moreMisses) {
532          // if there are misses beyond the last hit we extend the shaded area
533          missedecdf += iter.Current.Value;
534          if (row.Points.Count == 0 || row.Points.Last().XValue < iter.Current.Key) {
535            var dp = new DataPoint(iter.Current.Key, ecdf / movingTargets);
536            if (showMarkers) {
537              dp.MarkerStyle = MarkerStyle.Circle;
538              dp.MarkerBorderWidth = 1;
539              dp.MarkerSize = 5;
540            }
541            row.Points.Add(dp);
542            if (boundShadingCheckBox.Checked) {
543              rowShade.Points.Add(new DataPoint(iter.Current.Key, new[] { ecdf / totalTargets, (ecdf + missedecdf) / totalTargets }));
544            }
545          }
546          moreMisses = iter.MoveNext();
547
548          if (!labelPrinted && row.Points.Count > 0) {
549            var point = row.Points.Last();
550            if (showLabelsInTargetChart)
551              point.Label = row.Name;
552            point.MarkerStyle = MarkerStyle.Cross;
553            point.MarkerBorderWidth = 1;
554            point.MarkerSize = 10;
555            labelPrinted = true;
556          }
557        }
558
559        if (!labelPrinted && row.Points.Count > 0) {
560          var point = row.Points.Last();
561          if (showLabelsInTargetChart)
562            point.Label = row.Name;
563          point.MarkerStyle = MarkerStyle.Cross;
564          point.MarkerBorderWidth = 1;
565          point.MarkerSize = 10;
566          rowShade.Points.Add(new DataPoint(point.XValue, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
567          labelPrinted = true;
568        }
569
570        ConfigureSeries(row);
571        targetChart.Series.Add(rowShade);
572        targetChart.Series.Add(row);
573
574        colorCount = (colorCount + 1) % colors.Length;
575        if (colorCount == 0) lineStyleCount = (lineStyleCount + 1) % lineStyles.Length;
576      }
577    }
578
579    private void UpdateTargetChartAxisXBounds(double minEff, double maxEff) {
580      var minZeros = (int)Math.Floor(Math.Log10(minEff));
581      var maxZeros = (int)Math.Floor(Math.Log10(maxEff));
582      var axisMin = (decimal)Math.Pow(10, minZeros);
583      var axisMax = (decimal)Math.Pow(10, maxZeros);
584      if (!targetLogScalingCheckBox.Checked) {
585        var minAdd = (decimal)Math.Pow(10, minZeros - 1) * 2;
586        var maxAdd = (decimal)Math.Pow(10, maxZeros - 1) * 2;
587        while (axisMin + minAdd < (decimal)minEff) axisMin += minAdd;
588        while (axisMax <= (decimal)maxEff) axisMax += maxAdd;
589      } else axisMax = (decimal)Math.Pow(10, (int)Math.Ceiling(Math.Log10(maxEff)));
590      targetChart.ChartAreas[0].AxisX.Minimum = (double)axisMin;
591      targetChart.ChartAreas[0].AxisX.Maximum = (double)axisMax;
592    }
593
594    private IEnumerable<double> GetAbsoluteTargets(ProblemInstance pInstance) {
595      if (!targetsAreRelative) return targets;
596      if (!pInstance.Maximization.HasValue) throw new ArgumentException("Problem doesn't specify if it is to be maximized or minimized.");
597
598      var maximization = pInstance.Maximization.Value;
599      var bestKnown = pInstance.BestKnownQuality;
600      if (double.IsNaN(bestKnown)) throw new ArgumentException("Problem instance does not have a defined best - known quality.");
601      IEnumerable<double> tmp = null;
602      if (bestKnown > 0) {
603        tmp = targets.Select(x => (maximization ? (1 - x) : (1 + x)) * bestKnown);
604      } else if (bestKnown < 0) {
605        tmp = targets.Select(x => (!maximization ? (1 - x) : (1 + x)) * bestKnown);
606      } else {
607        // relative to 0 is impossible
608        tmp = targets;
609      }
610      return tmp;
611    }
612
613    private double[] GetAbsoluteTargetsWorstToBest(ProblemInstance pInstance) {
614      if (targetsAreRelative && double.IsNaN(pInstance.BestKnownQuality)) throw new ArgumentException("Problem instance does not have a defined best-known quality.");
615      if (!pInstance.Maximization.HasValue) throw new ArgumentException("Problem doesn't specify if it is to be maximized or minimized.");
616      var absTargets = GetAbsoluteTargets(pInstance);
617      return (pInstance.Maximization.Value
618        ? absTargets.OrderBy(x => x) : absTargets.OrderByDescending(x => x)).ToArray();
619    }
620
621    private void GenerateDefaultTargets() {
622      targets = new[] { 0.1, 0.095, 0.09, 0.085, 0.08, 0.075, 0.07, 0.065, 0.06, 0.055, 0.05, 0.045, 0.04, 0.035, 0.03, 0.025, 0.02, 0.015, 0.01, 0.005, 0 };
623      SynchronizeTargetTextBox();
624    }
625
626    private Tuple<bool, double> GetEffortToHitTarget(
627        ConvergenceGraph cgraph,
628        double absTarget, bool maximization) {
629      if (cgraph.Points == 0)
630        throw new ArgumentException("Convergence graph is empty.", "convergenceGraph");
631     
632      var index = cgraph.BinarySearch(new ConvergenceGraphPoint(0.0, absTarget), new TargetComparer(maximization));
633      if (index >= 0) {
634        return Tuple.Create(true, cgraph[index].Runlength);
635      } else {
636        index = ~index;
637        if (index >= cgraph.Points)
638          return Tuple.Create(false, cgraph.Last().Runlength);
639        return Tuple.Create(true, cgraph[index].Runlength);
640      }
641    }
642
643    private void UpdateErtTables() {
644      ertTableView.Content = null;
645      var columns = targets.Length + 1;
646      var totalRows = groups.Count * groups.Max(x => x.GetNumberOfCases()) + groups.Max(x => x.GetNumberOfCases());
647      var matrix = new StringMatrix(totalRows, columns);
648      var rowNames = new List<string>();
649      matrix.ColumnNames = targets.Select(x => targetsAreRelative ? (100 * x).ToString() + "%" : x.ToString())
650        .Concat(new[] { "#succ" }).ToList();
651      var rowCount = 0;
652
653      var tableName = (string)dataTableComboBox.SelectedItem;
654      if (string.IsNullOrEmpty(tableName)) return;
655     
656      var problems = groups.SelectMany(x => x.GetCases()).Distinct().ToList();
657
658      foreach (var problem in problems.OrderBy(x => x.ProblemName, new NaturalStringComparer())) {
659        if (double.IsNaN(problem.BestKnownQuality) || !problem.Maximization.HasValue) continue;
660        rowNames.Add(problem.ToString());
661        var max = problem.Maximization.Value;
662        var absTargets = GetAbsoluteTargetsWorstToBest(problem);
663        if (targetsAreRelative) {
664          // print out the absolute target values
665          for (var i = 0; i < absTargets.Length; i++) {
666            matrix[rowCount, i] = absTargets[i].ToString("##,0.0", CultureInfo.CurrentCulture.NumberFormat);
667          }
668        }
669        rowCount++;
670
671        foreach (var alg in groups) {
672          rowNames.Add(alg.Name);
673          var runs = alg.GetTrials(problem).ToList();
674          if (runs.Count == 0) {
675            matrix[rowCount, columns - 1] = "N/A";
676            rowCount++;
677            continue;
678          }
679          var result = default(ErtCalculationResult);
680          for (var i = 0; i < absTargets.Length; i++) {
681            result = ExpectedRuntimeHelper.CalculateErt(runs.Select(x => x.ToTuples()), absTargets[i], max);
682            matrix[rowCount, i] = result.ToString();
683          }
684          matrix[rowCount, columns - 1] = targets.Length > 0 ? result.SuccessfulRuns + "/" + result.TotalRuns : "-";
685          rowCount++;
686        }
687      }
688      matrix.RowNames = rowNames;
689      ertTableView.Content = matrix;
690      ertTableView.DataGridView.AutoResizeColumns(DataGridViewAutoSizeColumnsMode.AllCells);
691    }
692    #endregion
693
694    #region Performance analysis by (multiple) budget(s)
695    private void UpdateResultsByCost() {
696      // necessary to reset log scale -> empty chart cannot use log scaling
697      byCostDataTable.VisualProperties.XAxisLogScale = false;
698      byCostDataTable.Rows.Clear();
699
700      var table = (string)dataTableComboBox.SelectedItem;
701      if (string.IsNullOrEmpty(table)) return;
702
703      if (budgets == null) GenerateDefaultBudgets(table);
704     
705      if (groups.Count == 0) return;
706
707      var colorCount = 0;
708      var lineStyleCount = 0;
709     
710      foreach (var alg in groups) {
711        var hits = new Dictionary<string, SortedList<double, double>>();
712
713        foreach (var problem in alg.GetCases()) {
714          foreach (var run in alg.GetTrials(problem)) {
715            CalculateHitsForEachBudget(hits, run, problem, alg.Name);
716          }
717        }
718
719        foreach (var list in hits) {
720          var row = new IndexedDataRow<double>(list.Key) {
721            VisualProperties = {
722              ChartType = DataRowVisualProperties.DataRowChartType.StepLine,
723              LineWidth = 2,
724              Color = colors[colorCount],
725              LineStyle = hlLineStyles[lineStyleCount],
726              StartIndexZero = false
727            }
728          };
729
730          var total = 0.0;
731          var count = list.Value.Count;
732          foreach (var h in list.Value) {
733            total += h.Value;
734            row.Values.Add(Tuple.Create(h.Key, total / (double)count));
735          }
736
737          byCostDataTable.Rows.Add(row);
738        }
739        colorCount = (colorCount + 1) % colors.Length;
740        if (colorCount == 0) lineStyleCount = (lineStyleCount + 1) % lineStyles.Length;
741      }
742
743      byCostDataTable.VisualProperties.XAxisTitle = "Targets to Best-Known Ratio";
744      byCostDataTable.VisualProperties.XAxisLogScale = byCostDataTable.Rows.Count > 0 && budgetLogScalingCheckBox.Checked;
745    }
746
747    private void GenerateDefaultBudgets(string table) {
748      var runs = Content.Where(x => x.Results.ContainsKey(table) && x.Results[table] is IndexedDataTable<double>)
749                        .Select(x => (IndexedDataTable<double>)x.Results[table])
750                        .Where(x => x.Rows.Count > 0 && x.Rows.First().Values.Count > 0)
751                        .Select(x => x.Rows.First())
752                        .ToList();
753      if (runs.Count == 0) {
754        budgets = new double[0];
755        suppressBudgetsEvents = true;
756        budgetsTextBox.Text = string.Empty;
757        suppressBudgetsEvents = false;
758        return;
759      }
760
761      var min = runs.Select(x => x.Values.Select(y => y.Item1).Min()).Min();
762      var max = runs.Select(x => x.Values.Select(y => y.Item1).Max()).Max();
763      var points = 3;
764      budgets = Enumerable.Range(1, points).Select(x => min + (x / (double)points) * (max - min)).ToArray();
765      suppressBudgetsEvents = true;
766      budgetsTextBox.Text = string.Join(" ; ", budgets);
767      suppressBudgetsEvents = false;
768    }
769
770    private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, ConvergenceGraph cgraph, ProblemInstance problem, string groupName) {
771      var max = problem.Maximization.Value;
772      var prevIndex = 0;
773      foreach (var b in budgets) {
774        var key = groupName + "-" + b;
775        var index = cgraph.BinarySearch(prevIndex, cgraph.Points - prevIndex, new ConvergenceGraphPoint(b, 0.0), new CostComparer());
776        if (index < 0) {
777          index = ~index;
778          if (index >= cgraph.Points) break; // the run wasn't long enough to use up budget b (or any subsequent larger one)
779        }
780        if (!hits.ContainsKey(key)) hits.Add(key, new SortedList<double, double>());
781        var v = cgraph[index];
782        var relTgt = CalculateRelativeDifference(max, problem.BestKnownQuality, v.Quality) + 1;
783        if (hits[key].ContainsKey(relTgt))
784          hits[key][relTgt]++;
785        else hits[key][relTgt] = 1.0;
786        prevIndex = index;
787      }
788    }
789    #endregion
790
791    private void UpdateCaption() {
792      Caption = Content != null ? Content.OptimizerName + " RLD View" : ViewAttribute.GetViewName(GetType());
793    }
794
795    private void SynchronizeTargetTextBox() {
796      if (InvokeRequired) Invoke((Action)SynchronizeTargetTextBox);
797      else {
798        if (targetsAreRelative)
799          targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100)) + "%";
800        else targetsTextBox.Text = string.Join(" ; ", targets);
801      }
802    }
803
804    private void groupComboBox_SelectedIndexChanged(object sender, EventArgs e) {
805      if (updateInProgress) return;
806      try {
807        updateInProgress = true;
808        UpdateRuns();
809        SetEnabledStateOfControls();
810      } finally { updateInProgress = false; }
811    }
812    private void problemComboBox_SelectedIndexChanged(object sender, EventArgs e) {
813      if (updateInProgress) return;
814      try {
815        updateInProgress = true;
816        UpdateRuns();
817        SetEnabledStateOfControls();
818      } finally { updateInProgress = false; }
819    }
820    private void dataTableComboBox_SelectedIndexChanged(object sender, EventArgs e) {
821      if (updateInProgress) return;
822      try {
823        updateInProgress = true;
824        if (dataTableComboBox.SelectedIndex >= 0)
825          GenerateDefaultBudgets((string)dataTableComboBox.SelectedItem);
826        UpdateBestKnownQualities();
827        UpdateRuns();
828        SetEnabledStateOfControls();
829      } finally { updateInProgress = false; }
830    }
831
832    private void logScalingCheckBox_CheckedChanged(object sender, EventArgs e) {
833      UpdateResultsByTarget();
834      byCostDataTable.VisualProperties.XAxisLogScale = byCostDataTable.Rows.Count > 0 && budgetLogScalingCheckBox.Checked;
835    }
836
837    private void boundShadingCheckBox_CheckedChanged(object sender, EventArgs e) {
838      UpdateResultsByTarget();
839    }
840
841    #region Event handlers for target analysis
842    private void targetsTextBox_Validating(object sender, CancelEventArgs e) {
843      if (updateInProgress) return;
844      try {
845        updateInProgress = true;
846        var targetStrings = targetsTextBox.Text.Split(new[] { '%', ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
847        var targetList = new List<decimal>();
848        foreach (var ts in targetStrings) {
849          decimal t;
850          if (!decimal.TryParse(ts, out t)) {
851            errorProvider.SetError(targetsTextBox, "Not all targets can be parsed: " + ts);
852            e.Cancel = true;
853            return;
854          }
855          if (targetsAreRelative)
856            targetList.Add(t / 100);
857          else targetList.Add(t);
858        }
859        if (targetList.Count == 0) {
860          errorProvider.SetError(targetsTextBox, "Give at least one target value!");
861          e.Cancel = true;
862          return;
863        }
864        e.Cancel = false;
865        errorProvider.SetError(targetsTextBox, null);
866        targets = targetsAreRelative ? targetList.Select(x => (double)x).OrderByDescending(x => x).ToArray() : targetList.Select(x => (double)x).ToArray();
867
868        SynchronizeTargetTextBox();
869        UpdateResultsByTarget();
870        SetEnabledStateOfControls();
871
872      } finally { updateInProgress = false; }
873    }
874
875    private void aggregateTargetsCheckBox_CheckedChanged(object sender, EventArgs e) {
876      if (updateInProgress) return;
877      try {
878        updateInProgress = true;
879        SuspendRepaint();
880        UpdateResultsByTarget();
881      } finally {
882        updateInProgress = false;
883        ResumeRepaint(true);
884      }
885    }
886
887    private void relativeOrAbsoluteComboBox_SelectedIndexChanged(object sender, EventArgs e) {
888      if (updateInProgress) return;
889      try {
890        updateInProgress = true;
891        var pd = (ProblemInstance)problemComboBox.SelectedItem;
892        if (!double.IsNaN(pd.BestKnownQuality) && pd.Maximization.HasValue) {
893          var max = pd.Maximization.Value;
894          if (targetsAreRelative) targets = GetAbsoluteTargets(pd).ToArray();
895          else {
896            // Rounding to 5 digits since it's certainly appropriate for this application
897            if (pd.BestKnownQuality > 0) {
898              targets = targets.Select(x => Math.Round(max ? 1.0 - (x / pd.BestKnownQuality) : (x / pd.BestKnownQuality) - 1.0, 5)).ToArray();
899            } else if (pd.BestKnownQuality < 0) {
900              targets = targets.Select(x => Math.Round(!max ? 1.0 - (x / pd.BestKnownQuality) : (x / pd.BestKnownQuality) - 1.0, 5)).ToArray();
901            }
902          }
903        }
904        targetsAreRelative = (string)relativeOrAbsoluteComboBox.SelectedItem == "relative";
905        SynchronizeTargetTextBox();
906       
907        SuspendRepaint();
908        UpdateResultsByTarget();
909      } finally {
910        updateInProgress = false;
911        ResumeRepaint(true);
912      }
913    }
914
915    private void generateTargetsButton_Click(object sender, EventArgs e) {
916      if (targets == null) return;
917      decimal max = 10, min = 0, count = 10;
918      max = (decimal)targets.Max();
919      min = (decimal)targets.Min();
920      count = targets.Length - 1;
921      if (targetsAreRelative) {
922        max *= 100;
923        min *= 100;
924      }
925      using (var dialog = new DefineArithmeticProgressionDialog(false, min, max, (max - min) / count)) {
926        if (dialog.ShowDialog() == DialogResult.OK) {
927          if (dialog.Values.Any()) {
928            targets = targetsAreRelative
929              ? dialog.Values.OrderByDescending(x => x).Select(x => (double)x / 100.0).ToArray()
930              : dialog.Values.Select(x => (double)x).ToArray();
931
932            try {
933              updateInProgress = true;
934              SynchronizeTargetTextBox();
935              UpdateResultsByTarget();
936              SetEnabledStateOfControls();
937            } finally { updateInProgress = false; }
938          }
939        }
940      }
941    }
942
943    private void addTargetsAsResultButton_Click(object sender, EventArgs e) {
944      var table = (string)dataTableComboBox.SelectedItem;
945      if (string.IsNullOrEmpty(table)) return;
946     
947      foreach (var run in Content) {
948        if (!run.Results.ContainsKey(table) || !(run.Results[table] is IndexedDataTable<double>)) continue;
949        var resultsTable = (IndexedDataTable<double>)run.Results[table];
950        var values = resultsTable.Rows.First().Values;
951        var pd = new ProblemInstance(run);
952        if (!pd.Maximization.HasValue) continue;
953        pd = problems.Single(x => x.Equals(pd));
954        if (targetsAreRelative && double.IsNaN(pd.BestKnownQuality)) continue;
955
956        var max = pd.Maximization.Value;
957        var absTargets = GetAbsoluteTargetsWorstToBest(pd);
958        var cgraph = new ConvergenceGraph(resultsTable, max);
959
960        var prevIndex = 0;
961        for (var i = 0; i < absTargets.Length; i++) {
962          var absTarget = absTargets[i];
963          var index = cgraph.BinarySearch(prevIndex, values.Count - prevIndex, new ConvergenceGraphPoint(0.0, absTarget), new TargetComparer(max));
964          if (index < 0) {
965            index = ~index;
966            if (index >= values.Count) break; // the target (and subsequent ones) wasn't achieved
967          }
968          var target = targetsAreRelative ? (targets[i] * 100) : absTarget;
969          run.Results[table + (targetsAreRelative ? ".RelTarget " : ".AbsTarget ") + target + (targetsAreRelative ? "%" : string.Empty)] = new DoubleValue(values[index].Item1);
970          prevIndex = index;
971        }
972      }
973    }
974
975    private void markerCheckBox_CheckedChanged(object sender, EventArgs e) {
976      try {
977        updateInProgress = true;
978        SuspendRepaint();
979        UpdateResultsByTarget();
980      } finally {
981        updateInProgress = false;
982        ResumeRepaint(true);
983      }
984    }
985
986    private void showLabelsCheckBox_CheckedChanged(object sender, EventArgs e) {
987      showLabelsInTargetChart = showLabelsCheckBox.Checked;
988      try {
989        updateInProgress = true;
990        SuspendRepaint();
991        UpdateResultsByTarget();
992      } finally {
993        updateInProgress = false;
994        ResumeRepaint(true);
995      }
996    }
997    #endregion
998
999    #region Event handlers for cost analysis
1000    private bool suppressBudgetsEvents;
1001    private void budgetsTextBox_Validating(object sender, CancelEventArgs e) {
1002      if (suppressBudgetsEvents) return;
1003      var budgetStrings = budgetsTextBox.Text.Split(new[] { ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
1004      var budgetList = new List<double>();
1005      foreach (var ts in budgetStrings) {
1006        double b;
1007        if (!double.TryParse(ts, out b)) {
1008          errorProvider.SetError(budgetsTextBox, "Not all budgets can be parsed: " + ts);
1009          e.Cancel = true;
1010          return;
1011        }
1012        budgetList.Add(b);
1013      }
1014      if (budgetList.Count == 0) {
1015        errorProvider.SetError(budgetsTextBox, "Give at least one budget value!");
1016        e.Cancel = true;
1017        return;
1018      }
1019      e.Cancel = false;
1020      errorProvider.SetError(budgetsTextBox, null);
1021      budgets = budgetList.OrderBy(x => x).ToArray();
1022      try {
1023        suppressBudgetsEvents = true;
1024        budgetsTextBox.Text = string.Join(" ; ", budgets);
1025      } finally { suppressBudgetsEvents = false; }
1026      UpdateResultsByCost();
1027      SetEnabledStateOfControls();
1028    }
1029
1030    private void generateBudgetsButton_Click(object sender, EventArgs e) {
1031      decimal max = 1, min = 0, count = 10;
1032      if (budgets != null) {
1033        max = (decimal)budgets.Max();
1034        min = (decimal)budgets.Min();
1035        count = budgets.Length;
1036      } else if (Content.Count > 0 && dataTableComboBox.SelectedIndex >= 0) {
1037        var table = (string)dataTableComboBox.SelectedItem;
1038        min = (decimal)Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Min(y => y.Item1)).Min();
1039        max = (decimal)Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Max(y => y.Item1)).Max();
1040        count = 3;
1041      }
1042      using (var dialog = new DefineArithmeticProgressionDialog(false, min, max, (max - min) / count)) {
1043        if (dialog.ShowDialog() == DialogResult.OK) {
1044          if (dialog.Values.Any()) {
1045            budgets = dialog.Values.OrderBy(x => x).Select(x => (double)x).ToArray();
1046
1047            try {
1048              suppressBudgetsEvents = true;
1049              budgetsTextBox.Text = string.Join(" ; ", budgets);
1050            } finally { suppressBudgetsEvents = false; }
1051
1052            UpdateResultsByCost();
1053            SetEnabledStateOfControls();
1054          }
1055        }
1056      }
1057    }
1058
1059    private void addBudgetsAsResultButton_Click(object sender, EventArgs e) {
1060      var table = (string)dataTableComboBox.SelectedItem;
1061
1062      foreach (var run in Content) {
1063        if (!run.Results.ContainsKey(table)) continue;
1064        var resultsTable = (IndexedDataTable<double>)run.Results[table];
1065        var values = resultsTable.Rows.First().Values;
1066        var pd = new ProblemInstance(run);
1067        if (!pd.Maximization.HasValue) continue;
1068        pd = problems.Single(x => x.Equals(pd));
1069        var max = pd.Maximization.Value;
1070        var cgraph = new ConvergenceGraph(resultsTable, max);
1071        var prevIndex = 0;
1072        foreach (var b in budgets) {
1073          var index = cgraph.BinarySearch(prevIndex, values.Count - prevIndex, new ConvergenceGraphPoint(b, 0.0), new CostComparer());
1074          if (index < 0) {
1075            index = ~index;
1076            if (index >= values.Count) break; // the run wasn't long enough to use up budget b (or any subsequent larger one)
1077          }
1078          var v = values[index];
1079          var tgt = targetsAreRelative ? CalculateRelativeDifference(max, pd.BestKnownQuality, v.Item2) : v.Item2;
1080          run.Results[table + (targetsAreRelative ? ".CostForRelTarget " : ".CostForAbsTarget ") + b] = new DoubleValue(tgt);
1081          prevIndex = index;
1082        }
1083      }
1084    }
1085    #endregion
1086
1087    #region Helpers
1088    private double CalculateRelativeDifference(bool maximization, double bestKnown, double fit) {
1089      if (bestKnown == 0) {
1090        // no relative difference with respect to bestKnown possible
1091        return maximization ? -fit : fit;
1092      }
1093      var absDiff = (fit - bestKnown);
1094      var relDiff = absDiff / bestKnown;
1095      if (maximization) {
1096        return bestKnown > 0 ? -relDiff : relDiff;
1097      } else {
1098        return bestKnown > 0 ? relDiff : -relDiff;
1099      }
1100    }
1101
1102    private Dictionary<ProblemInstance, double> CalculateBestTargetPerProblemInstance(string table) {
1103      if (table == null) table = string.Empty;
1104      var dict = new Dictionary<ProblemInstance, double>();
1105      foreach (var k in from r in Content where r.Visible let pd = new ProblemInstance(r) group r by pd into g select new { Problem = g.Key, Runs = g.ToList() }) {
1106        var pd = k.Problem;
1107        if (!pd.Maximization.HasValue) continue;
1108        var values = GetQualityValues(k.Runs, table).ToList();
1109        var target = double.NaN;
1110        if (values.Count > 0) {
1111          target = pd.Maximization.Value ? values.Max() : values.Min();
1112        }
1113        dict[pd] = target;
1114      }
1115      return dict;
1116    }
1117
1118    private IEnumerable<double> GetQualityValues(List<IRun> runs, string table) {
1119      foreach (var r in runs) {
1120        IItem item;
1121        if (r.Parameters.TryGetValue("BestKnownQuality", out item)) {
1122          var dval = item as DoubleValue;
1123          if (dval != null && !double.IsNaN(dval.Value))
1124            yield return dval.Value;
1125        }
1126        if (r.Results.TryGetValue(table, out item)) {
1127          var dt = item as IndexedDataTable<double>;
1128          if (dt != null && dt.Rows.Count > 0 && dt.Rows.First().Values.Count > 0) {
1129            var last = dt.Rows.First().Values.Last().Item2;
1130            if (!double.IsNaN(last))
1131              yield return Math.Round(last, 10);
1132          }
1133        }
1134      }
1135    }
1136
1137    private void UpdateRuns() {
1138      if (InvokeRequired) {
1139        Invoke((Action)UpdateRuns);
1140        return;
1141      }
1142      SuspendRepaint();
1143      try {
1144        GroupRuns();
1145        UpdateResultsByTarget();
1146        UpdateResultsByCost();
1147      } finally { ResumeRepaint(true); }
1148    }
1149
1150    private void UpdateBestKnownQualities() {
1151      var table = (string)dataTableComboBox.SelectedItem;
1152      if (string.IsNullOrEmpty(table)) return;
1153
1154      var targetsPerProblem = CalculateBestTargetPerProblemInstance(table);
1155      foreach (var pd in problems) {
1156        double bkq;
1157        if (targetsPerProblem.TryGetValue(pd, out bkq))
1158          pd.BestKnownQuality = bkq;
1159        else pd.BestKnownQuality = double.NaN;
1160      }
1161    }
1162    #endregion
1163
1164    private void ConfigureSeries(Series series) {
1165      series.SmartLabelStyle.Enabled = showLabelsInTargetChart;
1166      series.SmartLabelStyle.AllowOutsidePlotArea = LabelOutsidePlotAreaStyle.No;
1167      series.SmartLabelStyle.CalloutLineAnchorCapStyle = LineAnchorCapStyle.None;
1168      series.SmartLabelStyle.CalloutLineColor = series.Color;
1169      series.SmartLabelStyle.CalloutLineWidth = 2;
1170      series.SmartLabelStyle.CalloutStyle = LabelCalloutStyle.Underlined;
1171      series.SmartLabelStyle.IsOverlappedHidden = false;
1172      series.SmartLabelStyle.MaxMovingDistance = 200;
1173      series.ToolTip = series.LegendText + " X = #VALX, Y = #VALY";
1174    }
1175
1176    private void chart_MouseDown(object sender, MouseEventArgs e) {
1177      HitTestResult result = targetChart.HitTest(e.X, e.Y);
1178      if (result.ChartElementType == ChartElementType.LegendItem) {
1179        ToggleTargetChartSeriesVisible(result.Series);
1180      }
1181    }
1182    private void chart_MouseMove(object sender, MouseEventArgs e) {
1183      HitTestResult result = targetChart.HitTest(e.X, e.Y);
1184      if (result.ChartElementType == ChartElementType.LegendItem)
1185        this.Cursor = Cursors.Hand;
1186      else
1187        this.Cursor = Cursors.Default;
1188    }
1189    private void chart_CustomizeLegend(object sender, CustomizeLegendEventArgs e) {
1190      foreach (LegendItem legendItem in e.LegendItems) {
1191        var series = targetChart.Series[legendItem.SeriesName];
1192        if (series != null) {
1193          bool seriesIsInvisible = invisibleTargetSeries.Any(x => x.Name == series.Name);
1194          foreach (LegendCell cell in legendItem.Cells) {
1195            cell.ForeColor = seriesIsInvisible ? Color.Gray : Color.Black;
1196          }
1197        }
1198      }
1199    }
1200
1201    private void ToggleTargetChartSeriesVisible(Series series) {
1202      var indexList = invisibleTargetSeries.FindIndex(x => x.Name == series.Name);
1203      var indexChart = targetChart.Series.IndexOf(series);
1204      if (targetChart.Series.Count == 1) targetChart.ChartAreas[0].AxisX.IsLogarithmic = false;
1205      targetChart.Series.RemoveAt(indexChart);
1206      var s = indexList >= 0 ? invisibleTargetSeries[indexList] : new Series(series.Name) {
1207        Color = series.Color,
1208        ChartType = series.ChartType,
1209        BorderWidth = series.BorderWidth,
1210        BorderDashStyle = series.BorderDashStyle
1211      };
1212      if (indexList < 0) {
1213        // hide
1214        invisibleTargetSeries.Add(series);
1215        var shadeSeries = targetChart.Series.FirstOrDefault(x => x.Name == series.Name + "-range");
1216        if (shadeSeries != null) {
1217          if (targetChart.Series.Count == 1) targetChart.ChartAreas[0].AxisX.IsLogarithmic = false;
1218          targetChart.Series.Remove(shadeSeries);
1219          invisibleTargetSeries.Add(shadeSeries);
1220          indexChart--;
1221        }
1222      } else {
1223        // show
1224        invisibleTargetSeries.RemoveAt(indexList);
1225        var shadeSeries = invisibleTargetSeries.FirstOrDefault(x => x.Name == series.Name + "-range");
1226        if (shadeSeries != null) {
1227          invisibleTargetSeries.Remove(shadeSeries);
1228          InsertOrAddSeries(indexChart, shadeSeries);
1229          indexChart++;
1230        }
1231      }
1232      InsertOrAddSeries(indexChart, s);
1233      targetChart.ChartAreas[0].AxisX.IsLogarithmic = CanDisplayLogarithmic();
1234    }
1235
1236    private bool CanDisplayLogarithmic() {
1237      return targetLogScalingCheckBox.Checked
1238        && targetChart.Series.Count > 0 // must have a series
1239        && targetChart.Series.Any(x => x.Points.Count > 0) // at least one series must have points
1240        && targetChart.Series.All(s => s.Points.All(p => p.XValue > 0)); // all points must be positive
1241    }
1242
1243    private void InsertOrAddSeries(int index, Series s) {
1244      if (targetChart.Series.Count <= index)
1245        targetChart.Series.Add(s);
1246      else targetChart.Series.Insert(index, s);
1247    }
1248
1249    #region Helper classes
1250    private class AlgorithmInstance : INotifyPropertyChanged {
1251      private string name;
1252      public string Name {
1253        get { return name; }
1254        set {
1255          if (name == value) return;
1256          name = value;
1257          OnPropertyChanged("Name");
1258        }
1259      }
1260
1261      private Dictionary<ProblemInstance, List<ConvergenceGraph>> performanceData;
1262
1263      public int GetNumberOfCases() {
1264        return performanceData.Count;
1265      }
1266
1267      public IEnumerable<ProblemInstance> GetCases() {
1268        return performanceData.Keys;
1269      }
1270
1271      public int GetNumberOfTrials(ProblemInstance p) {
1272        if (p == ProblemInstance.MatchAll) return performanceData.Select(x => x.Value.Count).Sum();
1273        List<ConvergenceGraph> trials;
1274        if (performanceData.TryGetValue(p, out trials))
1275          return trials.Count;
1276
1277        return 0;
1278      }
1279
1280      public IEnumerable<ConvergenceGraph> GetTrials(ProblemInstance p) {
1281        if (p == ProblemInstance.MatchAll) return performanceData.SelectMany(x => x.Value);
1282        List<ConvergenceGraph> trials;
1283        if (performanceData.TryGetValue(p, out trials))
1284          return trials;
1285
1286        return Enumerable.Empty<ConvergenceGraph>();
1287      }
1288
1289      public AlgorithmInstance(string name, IEnumerable<IGrouping<ProblemInstance, IndexedDataTable<double>>> trials) {
1290        this.name = name;
1291
1292        performanceData = new Dictionary<ProblemInstance, List<ConvergenceGraph>>();
1293        foreach (var t in trials) {
1294          if (double.IsNaN(t.Key.BestKnownQuality) || !t.Key.Maximization.HasValue) continue;
1295          performanceData[t.Key] = t.Select(c => new ConvergenceGraph(c, t.Key.Maximization.Value)).ToList();
1296        }
1297      }
1298
1299      public override bool Equals(object obj) {
1300        var other = obj as AlgorithmInstance;
1301        if (other == null) return false;
1302        return name == other.name;
1303      }
1304
1305      public override int GetHashCode() {
1306        return name.GetHashCode();
1307      }
1308
1309      public event PropertyChangedEventHandler PropertyChanged;
1310      protected virtual void OnPropertyChanged(string propertyName = null) {
1311        var handler = PropertyChanged;
1312        if (handler != null) handler(this, new PropertyChangedEventArgs(propertyName));
1313      }
1314    }
1315
1316    private class ProblemInstance : INotifyPropertyChanged {
1317      private readonly bool matchAll;
1318      public static readonly ProblemInstance MatchAll = new ProblemInstance() {
1319        ProblemName = "All with Best-Known"
1320      };
1321
1322      private ProblemInstance() {
1323        ProblemType = string.Empty;
1324        ProblemName = string.Empty;
1325        Evaluator = string.Empty;
1326        Maximization = null;
1327        DisplayProblemType = false;
1328        DisplayProblemName = false;
1329        DisplayEvaluator = false;
1330        DisplayMaximization = false;
1331        matchAll = true;
1332        BestKnownQuality = double.NaN;
1333      }
1334
1335      public ProblemInstance(IRun run) {
1336        ProblemType = GetStringValueOrEmpty(run, "Problem Type");
1337        ProblemName = GetStringValueOrEmpty(run, "Problem Name");
1338        Evaluator = GetStringValueOrEmpty(run, "Evaluator");
1339        Maximization = GetBoolValueOrEmpty(run, "Maximization");
1340        DisplayProblemType = !string.IsNullOrEmpty(ProblemType);
1341        DisplayProblemName = !string.IsNullOrEmpty(ProblemName);
1342        DisplayEvaluator = !string.IsNullOrEmpty(Evaluator);
1343        DisplayMaximization = Maximization.HasValue;
1344        matchAll = false;
1345        BestKnownQuality = GetDoubleValueOrNaN(run, "BestKnownQuality");
1346      }
1347
1348      private bool displayProblemType;
1349      public bool DisplayProblemType {
1350        get { return displayProblemType; }
1351        set {
1352          if (displayProblemType == value) return;
1353          displayProblemType = value;
1354          OnPropertyChanged("DisplayProblemType");
1355        }
1356      }
1357      private string problemType;
1358      public string ProblemType {
1359        get { return problemType; }
1360        set {
1361          if (problemType == value) return;
1362          problemType = value;
1363          OnPropertyChanged("ProblemType");
1364        }
1365      }
1366      private bool displayProblemName;
1367      public bool DisplayProblemName {
1368        get { return displayProblemName; }
1369        set {
1370          if (displayProblemName == value) return;
1371          displayProblemName = value;
1372          OnPropertyChanged("DisplayProblemName");
1373        }
1374      }
1375      private string problemName;
1376      public string ProblemName {
1377        get { return problemName; }
1378        set {
1379          if (problemName == value) return;
1380          problemName = value;
1381          OnPropertyChanged("ProblemName");
1382        }
1383      }
1384      private bool displayEvaluator;
1385      public bool DisplayEvaluator {
1386        get { return displayEvaluator; }
1387        set {
1388          if (displayEvaluator == value) return;
1389          displayEvaluator = value;
1390          OnPropertyChanged("DisplayEvaluator");
1391        }
1392      }
1393      private string evaluator;
1394      public string Evaluator {
1395        get { return evaluator; }
1396        set {
1397          if (evaluator == value) return;
1398          evaluator = value;
1399          OnPropertyChanged("Evaluator");
1400        }
1401      }
1402      private bool displayMaximization;
1403      public bool DisplayMaximization {
1404        get { return displayMaximization; }
1405        set {
1406          if (displayMaximization == value) return;
1407          displayMaximization = value;
1408          OnPropertyChanged("DisplayMaximization");
1409        }
1410      }
1411      private bool? maximization;
1412      public bool? Maximization {
1413        get { return maximization; }
1414        set {
1415          if (maximization == value) return;
1416          maximization = value;
1417          OnPropertyChanged("Maximization");
1418        }
1419      }
1420      private double bestKnownQuality;
1421      public double BestKnownQuality {
1422        get { return bestKnownQuality; }
1423        set {
1424          if (bestKnownQuality == value) return;
1425          bestKnownQuality = value;
1426          OnPropertyChanged("BestKnownQuality");
1427        }
1428      }
1429
1430      public bool Match(IRun run) {
1431        return matchAll ||
1432               (GetStringValueOrEmpty(run, "Problem Type") == ProblemType
1433               && GetStringValueOrEmpty(run, "Problem Name") == ProblemName
1434               && GetStringValueOrEmpty(run, "Evaluator") == Evaluator
1435               && GetBoolValueOrEmpty(run, "Maximization") == Maximization);
1436      }
1437
1438      private double GetDoubleValueOrNaN(IRun run, string key) {
1439        IItem param;
1440        if (run.Parameters.TryGetValue(key, out param)) {
1441          var dv = param as DoubleValue;
1442          return dv != null ? dv.Value : double.NaN;
1443        }
1444        return double.NaN;
1445      }
1446
1447      private string GetStringValueOrEmpty(IRun run, string key) {
1448        IItem param;
1449        if (run.Parameters.TryGetValue(key, out param)) {
1450          var sv = param as StringValue;
1451          return sv != null ? sv.Value : string.Empty;
1452        }
1453        return string.Empty;
1454      }
1455
1456      private bool? GetBoolValueOrEmpty(IRun run, string key) {
1457        IItem param;
1458        if (run.Parameters.TryGetValue(key, out param)) {
1459          var bv = param as BoolValue;
1460          if (bv != null) return bv.Value;
1461        }
1462        return null;
1463      }
1464
1465      public override bool Equals(object obj) {
1466        var other = obj as ProblemInstance;
1467        if (other == null) return false;
1468        return ProblemType == other.ProblemType
1469               && ProblemName == other.ProblemName
1470               && Evaluator == other.Evaluator
1471               && Maximization == other.Maximization;
1472      }
1473
1474      public override int GetHashCode() {
1475        return ProblemType.GetHashCode() ^ ProblemName.GetHashCode() ^ Evaluator.GetHashCode() ^ Maximization.GetHashCode();
1476      }
1477
1478      public override string ToString() {
1479        return string.Join("  --  ", new[] {
1480          (DisplayProblemType ? ProblemType : string.Empty),
1481          (DisplayProblemName ? ProblemName : string.Empty),
1482          (DisplayEvaluator ? Evaluator : string.Empty),
1483          (DisplayMaximization && Maximization.HasValue ? (Maximization.Value ? "MAX" : "MIN") : string.Empty),
1484          !double.IsNaN(BestKnownQuality) ? BestKnownQuality.ToString(CultureInfo.CurrentCulture.NumberFormat) : string.Empty }.Where(x => !string.IsNullOrEmpty(x)));
1485      }
1486
1487      public event PropertyChangedEventHandler PropertyChanged;
1488      protected virtual void OnPropertyChanged(string propertyName = null) {
1489        var handler = PropertyChanged;
1490        if (handler != null) handler(this, new PropertyChangedEventArgs(propertyName));
1491      }
1492    }
1493
1494    private class CostComparer : Comparer<ConvergenceGraphPoint> {
1495      public override int Compare(ConvergenceGraphPoint x, ConvergenceGraphPoint y) {
1496        return x.Runlength.CompareTo(y.Runlength);
1497      }
1498    }
1499
1500    private class TargetComparer : Comparer<ConvergenceGraphPoint> {
1501      public bool Maximization { get; private set; }
1502      public TargetComparer(bool maximization) {
1503        Maximization = maximization;
1504      }
1505
1506      public override int Compare(ConvergenceGraphPoint x, ConvergenceGraphPoint y) {
1507        return Maximization ? x.Quality.CompareTo(y.Quality) : y.Quality.CompareTo(x.Quality);
1508      }
1509    }
1510
1511    private class ConvergenceGraph : IEnumerable<ConvergenceGraphPoint> {
1512      private List<ConvergenceGraphPoint> data;
1513      private bool maximization;
1514      private string xAxisLabel;
1515
1516      public string XAxisName { get { return xAxisLabel; } }
1517
1518      public int Points {
1519        get { return data.Count; }
1520      }
1521
1522      public double TotalRunlength {
1523        get { return data.Last().Runlength; }
1524      }
1525
1526      public double BestQuality {
1527        get { return data.Last().Quality; }
1528      }
1529
1530      public double QualityAt(double runlength) {
1531        var point = data.SkipWhile(x => x.Runlength < runlength).FirstOrDefault();
1532        if (point == null) return double.NaN;
1533        return point.Quality;
1534      }
1535
1536      public double RunlengthFor(double quality) {
1537        var point = (maximization ? data.SkipWhile(x => x.Quality < quality) : data.SkipWhile(x => x.Quality > quality)).FirstOrDefault();
1538        if (point == null) return double.NaN;
1539        return point.Runlength;
1540      }
1541
1542      public ConvergenceGraphPoint this[int point] {
1543        get { return data[point]; }
1544      }
1545
1546      public ConvergenceGraph(IndexedDataTable<double> table, bool maximization) {
1547        data = table.Rows.First().Values.Select(x => new ConvergenceGraphPoint() { Runlength = x.Item1, Quality = x.Item2 }).ToList();
1548        xAxisLabel = table.VisualProperties.XAxisTitle;
1549        this.maximization = maximization;
1550      }
1551
1552      public IEnumerator<ConvergenceGraphPoint> GetEnumerator() {
1553        return data.GetEnumerator();
1554      }
1555
1556      IEnumerator IEnumerable.GetEnumerator() {
1557        return data.GetEnumerator();
1558      }
1559
1560      public int BinarySearch(ConvergenceGraphPoint item, IComparer<ConvergenceGraphPoint> comparer) {
1561        return data.BinarySearch(item, comparer);
1562      }
1563
1564      public int BinarySearch(int index, int count, ConvergenceGraphPoint point, IComparer<ConvergenceGraphPoint> comparer) {
1565        return data.BinarySearch(index, count, point, comparer);
1566      }
1567
1568      public IEnumerable<Tuple<double, double>> ToTuples() {
1569        return data.Select(x => Tuple.Create(x.Runlength, x.Quality));
1570      }
1571    }
1572
1573    private class ConvergenceGraphPoint {
1574      public double Runlength { get; set; }
1575      public double Quality { get; set; }
1576
1577      public ConvergenceGraphPoint() {
1578
1579      }
1580      public ConvergenceGraphPoint(double runlength, double quality) {
1581        Runlength = runlength;
1582        Quality = quality;
1583      }
1584    }
1585    #endregion
1586  }
1587}
Note: See TracBrowser for help on using the repository browser.