source: stable/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs @ 15584

Last change on this file since 15584 was 15584, checked in by swagner, 3 years ago

#2640: Updated year of copyrights in license headers on stable

File size: 57.9 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.ComponentModel;
25using System.Drawing;
26using System.Globalization;
27using System.Linq;
28using System.Windows.Forms;
29using System.Windows.Forms.DataVisualization.Charting;
30using HeuristicLab.Analysis;
31using HeuristicLab.Collections;
32using HeuristicLab.Core;
33using HeuristicLab.Core.Views;
34using HeuristicLab.Data;
35using HeuristicLab.MainForm;
36using HeuristicLab.MainForm.WindowsForms;
37
38namespace HeuristicLab.Optimization.Views {
39  [View("Run-length Distribution View")]
40  [Content(typeof(RunCollection), false)]
41  public partial class RunCollectionRLDView : ItemView {
42    private List<Series> invisibleTargetSeries;
43
44    private const string AllInstances = "All Instances";
45
46    private static readonly Color[] colors = new[] {
47      Color.FromArgb(0x40, 0x6A, 0xB7),
48      Color.FromArgb(0xB1, 0x6D, 0x01),
49      Color.FromArgb(0x4E, 0x8A, 0x06),
50      Color.FromArgb(0x75, 0x50, 0x7B),
51      Color.FromArgb(0x72, 0x9F, 0xCF),
52      Color.FromArgb(0xA4, 0x00, 0x00),
53      Color.FromArgb(0xAD, 0x7F, 0xA8),
54      Color.FromArgb(0x29, 0x50, 0xCF),
55      Color.FromArgb(0x90, 0xB0, 0x60),
56      Color.FromArgb(0xF5, 0x89, 0x30),
57      Color.FromArgb(0x55, 0x57, 0x53),
58      Color.FromArgb(0xEF, 0x59, 0x59),
59      Color.FromArgb(0xED, 0xD4, 0x30),
60      Color.FromArgb(0x63, 0xC2, 0x16),
61    };
62    private static readonly ChartDashStyle[] lineStyles = new[] {
63      ChartDashStyle.Solid,
64      ChartDashStyle.Dash,
65      ChartDashStyle.DashDot,
66      ChartDashStyle.Dot
67    };
68    private static readonly DataRowVisualProperties.DataRowLineStyle[] hlLineStyles = new[] {
69      DataRowVisualProperties.DataRowLineStyle.Solid,
70      DataRowVisualProperties.DataRowLineStyle.Dash,
71      DataRowVisualProperties.DataRowLineStyle.DashDot,
72      DataRowVisualProperties.DataRowLineStyle.Dot
73    };
74
75    public new RunCollection Content {
76      get { return (RunCollection)base.Content; }
77      set { base.Content = value; }
78    }
79
80    private double[] targets;
81    private double[] budgets;
82    private bool targetsAreRelative = true;
83    private bool showLabelsInTargetChart = true;
84    private readonly BindingList<ProblemInstance> problems;
85
86    private bool suppressUpdates;
87    private readonly IndexedDataTable<double> byCostDataTable;
88    public IndexedDataTable<double> ByCostDataTable {
89      get { return byCostDataTable; }
90    }
91
92    public RunCollectionRLDView() {
93      InitializeComponent();
94      invisibleTargetSeries = new List<Series>();
95
96      targetChart.CustomizeAllChartAreas();
97      targetChart.ChartAreas[0].CursorX.Interval = 1;
98      targetChart.SuppressExceptions = true;
99      byCostDataTable = new IndexedDataTable<double>("ECDF by Cost", "A data table containing the ECDF of function values (relative to best-known).") {
100        VisualProperties = {
101          YAxisTitle = "Proportion of runs",
102          YAxisMinimumFixedValue = 0,
103          YAxisMinimumAuto = false,
104          YAxisMaximumFixedValue = 1,
105          YAxisMaximumAuto = false
106        }
107      };
108      byCostViewHost.Content = byCostDataTable;
109      suppressUpdates = true;
110      relativeOrAbsoluteComboBox.SelectedItem = targetsAreRelative ? "relative" : "absolute";
111      problems = new BindingList<ProblemInstance>();
112      problemComboBox.DataSource = new BindingSource() { DataSource = problems };
113      problemComboBox.DataBindings.DefaultDataSourceUpdateMode = DataSourceUpdateMode.OnPropertyChanged;
114      suppressUpdates = false;
115    }
116
117    #region Content events
118    protected override void RegisterContentEvents() {
119      base.RegisterContentEvents();
120      Content.ItemsAdded += Content_ItemsAdded;
121      Content.ItemsRemoved += Content_ItemsRemoved;
122      Content.CollectionReset += Content_CollectionReset;
123      Content.UpdateOfRunsInProgressChanged += Content_UpdateOfRunsInProgressChanged;
124      Content.OptimizerNameChanged += Content_AlgorithmNameChanged;
125    }
126    protected override void DeregisterContentEvents() {
127      Content.ItemsAdded -= Content_ItemsAdded;
128      Content.ItemsRemoved -= Content_ItemsRemoved;
129      Content.CollectionReset -= Content_CollectionReset;
130      Content.UpdateOfRunsInProgressChanged -= Content_UpdateOfRunsInProgressChanged;
131      Content.OptimizerNameChanged -= Content_AlgorithmNameChanged;
132      base.DeregisterContentEvents();
133    }
134
135    private void Content_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IRun> e) {
136      if (suppressUpdates) return;
137      if (InvokeRequired) {
138        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_ItemsAdded), sender, e);
139        return;
140      }
141      UpdateGroupAndProblemComboBox();
142      UpdateDataTableComboBox();
143      foreach (var run in e.Items)
144        RegisterRunEvents(run);
145    }
146    private void Content_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IRun> e) {
147      if (suppressUpdates) return;
148      if (InvokeRequired) {
149        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_ItemsRemoved), sender, e);
150        return;
151      }
152      UpdateGroupAndProblemComboBox();
153      UpdateDataTableComboBox();
154      foreach (var run in e.Items)
155        DeregisterRunEvents(run);
156    }
157    private void Content_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
158      if (suppressUpdates) return;
159      if (InvokeRequired) {
160        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset), sender, e);
161        return;
162      }
163      UpdateGroupAndProblemComboBox();
164      UpdateDataTableComboBox();
165      foreach (var run in e.OldItems)
166        DeregisterRunEvents(run);
167    }
168    private void Content_AlgorithmNameChanged(object sender, EventArgs e) {
169      if (InvokeRequired)
170        Invoke(new EventHandler(Content_AlgorithmNameChanged), sender, e);
171      else UpdateCaption();
172    }
173    private void Content_UpdateOfRunsInProgressChanged(object sender, EventArgs e) {
174      if (InvokeRequired) {
175        Invoke(new EventHandler(Content_UpdateOfRunsInProgressChanged), sender, e);
176        return;
177      }
178      suppressUpdates = Content.UpdateOfRunsInProgress;
179      if (!suppressUpdates) {
180        UpdateDataTableComboBox();
181        UpdateGroupAndProblemComboBox();
182        UpdateRuns();
183      }
184    }
185
186    private void RegisterRunEvents(IRun run) {
187      run.PropertyChanged += run_PropertyChanged;
188    }
189    private void DeregisterRunEvents(IRun run) {
190      run.PropertyChanged -= run_PropertyChanged;
191    }
192    private void run_PropertyChanged(object sender, PropertyChangedEventArgs e) {
193      if (suppressUpdates) return;
194      if (InvokeRequired) {
195        Invoke((Action<object, PropertyChangedEventArgs>)run_PropertyChanged, sender, e);
196      } else {
197        if (e.PropertyName == "Visible")
198          UpdateRuns();
199      }
200    }
201    #endregion
202
203    protected override void OnContentChanged() {
204      base.OnContentChanged();
205      dataTableComboBox.Items.Clear();
206      groupComboBox.Items.Clear();
207      targetChart.ChartAreas[0].AxisX.IsLogarithmic = false;
208      targetChart.Series.Clear();
209      invisibleTargetSeries.Clear();
210      byCostDataTable.VisualProperties.XAxisLogScale = false;
211      byCostDataTable.Rows.Clear();
212
213      UpdateCaption();
214      if (Content != null) {
215        UpdateGroupAndProblemComboBox();
216        UpdateDataTableComboBox();
217      }
218    }
219
220
221    private void UpdateGroupAndProblemComboBox() {
222      var selectedGroupItem = (string)groupComboBox.SelectedItem;
223
224      var groupings = Content.ParameterNames.OrderBy(x => x).ToArray();
225      groupComboBox.Items.Clear();
226      groupComboBox.Items.Add(AllInstances);
227      groupComboBox.Items.AddRange(groupings);
228      if (selectedGroupItem != null && groupComboBox.Items.Contains(selectedGroupItem)) {
229        groupComboBox.SelectedItem = selectedGroupItem;
230      } else if (groupComboBox.Items.Count > 0) {
231        groupComboBox.SelectedItem = groupComboBox.Items[0];
232      }
233
234      var table = (string)dataTableComboBox.SelectedItem;
235      var problemDict = CalculateBestTargetPerProblemInstance(table);
236
237      var problemTypesDifferent = problemDict.Keys.Select(x => x.ProblemType).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
238      var problemNamesDifferent = problemDict.Keys.Select(x => x.ProblemName).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
239      var evaluatorDifferent = problemDict.Keys.Select(x => x.Evaluator).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
240      var maximizationDifferent = problemDict.Keys.Select(x => x.Maximization).Distinct().Count() > 1;
241      var allEqual = !problemTypesDifferent && !problemNamesDifferent && !evaluatorDifferent && !maximizationDifferent;
242
243      var selectedProblemItem = (ProblemInstance)problemComboBox.SelectedItem;
244      problemComboBox.DataSource = null;
245      problemComboBox.Items.Clear();
246      problems.Clear();
247      problems.Add(ProblemInstance.MatchAll);
248      problemComboBox.DataSource = new BindingSource() { DataSource = problems };
249      problemComboBox.DataBindings.DefaultDataSourceUpdateMode = DataSourceUpdateMode.OnPropertyChanged;
250      if (problems[0].Equals(selectedProblemItem)) problemComboBox.SelectedItem = problems[0];
251      foreach (var p in problemDict.ToList()) {
252        p.Key.BestKnownQuality = p.Value;
253        p.Key.DisplayProblemType = problemTypesDifferent;
254        p.Key.DisplayProblemName = problemNamesDifferent || allEqual;
255        p.Key.DisplayEvaluator = evaluatorDifferent;
256        p.Key.DisplayMaximization = maximizationDifferent;
257        problems.Add(p.Key);
258        if (p.Key.Equals(selectedProblemItem)) problemComboBox.SelectedItem = p.Key;
259      }
260      SetEnabledStateOfControls();
261    }
262
263    private void UpdateDataTableComboBox() {
264      string selectedItem = (string)dataTableComboBox.SelectedItem;
265
266      dataTableComboBox.Items.Clear();
267      var dataTables = (from run in Content
268                        from result in run.Results
269                        where result.Value is IndexedDataTable<double>
270                        select result.Key).Distinct().ToArray();
271
272      dataTableComboBox.Items.AddRange(dataTables);
273      if (selectedItem != null && dataTableComboBox.Items.Contains(selectedItem)) {
274        dataTableComboBox.SelectedItem = selectedItem;
275      } else if (dataTableComboBox.Items.Count > 0) {
276        dataTableComboBox.SelectedItem = dataTableComboBox.Items[0];
277      }
278    }
279
280    protected override void SetEnabledStateOfControls() {
281      base.SetEnabledStateOfControls();
282      groupComboBox.Enabled = Content != null;
283      problemComboBox.Enabled = Content != null && problemComboBox.Items.Count > 1;
284      dataTableComboBox.Enabled = Content != null && dataTableComboBox.Items.Count > 1;
285      addTargetsAsResultButton.Enabled = Content != null && targets != null && dataTableComboBox.SelectedIndex >= 0;
286      addBudgetsAsResultButton.Enabled = Content != null && budgets != null && dataTableComboBox.SelectedIndex >= 0;
287      generateTargetsButton.Enabled = targets != null;
288    }
289
290    private IEnumerable<AlgorithmInstance> GroupRuns() {
291      var table = (string)dataTableComboBox.SelectedItem;
292      if (string.IsNullOrEmpty(table)) yield break;
293
294      var selectedGroup = (string)groupComboBox.SelectedItem;
295      if (string.IsNullOrEmpty(selectedGroup)) yield break;
296
297      var selectedProblem = (ProblemInstance)problemComboBox.SelectedItem;
298      if (selectedProblem == null) yield break;
299     
300      foreach (var x in (from r in Content
301                         where (selectedGroup == AllInstances || r.Parameters.ContainsKey(selectedGroup))
302                           && selectedProblem.Match(r)
303                           && r.Results.ContainsKey(table)
304                           && r.Visible
305                         let key = selectedGroup == AllInstances ? AllInstances : r.Parameters[selectedGroup].ToString()
306                         group r by key into g
307                         select new AlgorithmInstance(g.Key, g, problems))) {
308        yield return x;
309      }
310    }
311
312    #region Performance analysis by (multiple) target(s)
313    private void UpdateResultsByTarget() {
314      // necessary to reset log scale -> empty chart cannot use log scaling
315      targetChart.ChartAreas[0].AxisX.IsLogarithmic = false;
316      targetChart.Series.Clear();
317      invisibleTargetSeries.Clear();
318     
319      var table = (string)dataTableComboBox.SelectedItem;
320      if (string.IsNullOrEmpty(table)) return;
321
322      if (targets == null) GenerateDefaultTargets();
323
324      var algInstances = GroupRuns().ToList();
325      if (algInstances.Count == 0) return;
326
327      var xAxisTitles = new HashSet<string>();
328
329      // hits describes the number of target hits at a certain time for a certain group
330      var hits = new Dictionary<string, SortedList<double, int>>();
331      // misses describes the number of target misses after a certain time for a certain group
332      // for instance when a run ends, but has not achieved all targets, misses describes
333      // how many targets have been left open at the point when the run ended
334      var misses = new Dictionary<string, SortedList<double, int>>();
335      var totalRuns = new Dictionary<string, int>();
336
337      var aggregate = aggregateTargetsCheckBox.Checked;
338      double minEff = double.MaxValue, maxEff = double.MinValue;
339      foreach (var alg in algInstances) {
340        var noRuns = 0;
341        SortedList<double, int> epdfHits = null, epdfMisses = null;
342        if (aggregate) {
343          hits[alg.Name] = epdfHits = new SortedList<double, int>();
344          misses[alg.Name] = epdfMisses = new SortedList<double, int>();
345        }
346        foreach (var problem in alg.GetProblemInstances()) {
347          var max = problem.IsMaximization();
348          var absTargets = GetAbsoluteTargets(problem).ToArray();
349          foreach (var run in alg.GetRuns(problem)) {
350            noRuns++;
351            var resultsTable = (IndexedDataTable<double>)run.Results[table];
352            xAxisTitles.Add(resultsTable.VisualProperties.XAxisTitle);
353
354            var efforts = absTargets.Select(t => GetEffortToHitTarget(resultsTable.Rows.First().Values, t, max)).ToArray();
355            minEff = Math.Min(minEff, efforts.Min(x => x.Item2));
356            maxEff = Math.Max(maxEff, efforts.Max(x => x.Item2));
357            for (var idx = 0; idx < efforts.Length; idx++) {
358              var e = efforts[idx];
359              if (!aggregate) {
360                var key = alg.Name + "@" + (targetsAreRelative
361                            ? (targets[idx] * 100).ToString(CultureInfo.CurrentCulture.NumberFormat) + "%"
362                            : targets[idx].ToString(CultureInfo.CurrentCulture.NumberFormat));
363                if (!hits.TryGetValue(key, out epdfHits))
364                  hits[key] = epdfHits = new SortedList<double, int>();
365                if (!misses.TryGetValue(key, out epdfMisses))
366                  misses[key] = epdfMisses = new SortedList<double, int>();
367                totalRuns[key] = noRuns;
368              };
369              var list = e.Item1 ? epdfHits : epdfMisses;
370              int v;
371              if (list.TryGetValue(e.Item2, out v))
372                list[e.Item2] = v + 1;
373              else list[e.Item2] = 1;
374            }
375          }
376        }
377        if (aggregate) totalRuns[alg.Name] = noRuns;
378      }
379
380      UpdateTargetChartAxisXBounds(minEff, maxEff);
381
382      DrawTargetsEcdf(hits, misses, totalRuns);
383
384      if (targets.Length == 1) {
385        if (targetsAreRelative)
386          targetChart.ChartAreas[0].AxisY.Title = "Probability to be " + (targets[0] * 100) + "% worse than best";
387        else targetChart.ChartAreas[0].AxisY.Title = "Probability to reach at least a fitness of " + targets[0];
388      } else targetChart.ChartAreas[0].AxisY.Title = "Proportion of reached targets";
389      targetChart.ChartAreas[0].AxisX.Title = string.Join(" / ", xAxisTitles);
390      targetChart.ChartAreas[0].AxisX.IsLogarithmic = CanDisplayLogarithmic();
391      targetChart.ChartAreas[0].CursorY.Interval = 0.05;
392
393      UpdateErtTables(algInstances);
394    }
395
396    private void DrawTargetsEcdf(Dictionary<string, SortedList<double, int>> hits, Dictionary<string, SortedList<double, int>> misses, Dictionary<string, int> noRuns) {
397      var colorCount = 0;
398      var lineStyleCount = 0;
399     
400      var showMarkers = markerCheckBox.Checked;
401      foreach (var list in hits) {
402        var row = new Series(list.Key) {
403          ChartType = SeriesChartType.StepLine,
404          BorderWidth = 3,
405          Color = colors[colorCount],
406          BorderDashStyle = lineStyles[lineStyleCount],
407        };
408        var rowShade = new Series(list.Key + "-range") {
409          IsVisibleInLegend = false,
410          ChartType = SeriesChartType.Range,
411          Color = Color.FromArgb(32, colors[colorCount]),
412          YValuesPerPoint = 2
413        };
414
415        var ecdf = 0.0;
416        var missedecdf = 0.0;
417        var iter = misses[list.Key].GetEnumerator();
418        var moreMisses = iter.MoveNext();
419        var totalTargets = noRuns[list.Key];
420        if (aggregateTargetsCheckBox.Checked) totalTargets *= targets.Length;
421        var movingTargets = totalTargets;
422        var labelPrinted = false;
423        foreach (var h in list.Value) {
424          var prevmissedecdf = missedecdf;
425          while (moreMisses && iter.Current.Key <= h.Key) {
426            if (!labelPrinted && row.Points.Count > 0) {
427              var point = row.Points.Last();
428              if (showLabelsInTargetChart)
429                point.Label = row.Name;
430              point.MarkerStyle = MarkerStyle.Cross;
431              point.MarkerBorderWidth = 1;
432              point.MarkerSize = 10;
433              labelPrinted = true;
434              rowShade.Points.Add(new DataPoint(point.XValue, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
435            }
436            missedecdf += iter.Current.Value;
437            movingTargets -= iter.Current.Value;
438            if (row.Points.Count > 0 && row.Points.Last().XValue == iter.Current.Key) {
439              row.Points.Last().SetValueY(ecdf / movingTargets);
440              row.Points.Last().Color = Color.FromArgb(255 - (int)Math.Floor(255 * (prevmissedecdf / totalTargets)), colors[colorCount]);
441            } else {
442              var dp = new DataPoint(iter.Current.Key, ecdf / movingTargets) {
443                Color = Color.FromArgb(255 - (int)Math.Floor(255 * (prevmissedecdf / totalTargets)), colors[colorCount])
444              };
445              if (showMarkers) {
446                dp.MarkerStyle = MarkerStyle.Circle;
447                dp.MarkerBorderWidth = 1;
448                dp.MarkerSize = 5;
449              }
450              row.Points.Add(dp);
451              prevmissedecdf = missedecdf;
452            }
453            if (boundShadingCheckBox.Checked) {
454              if (rowShade.Points.Count > 0 && rowShade.Points.Last().XValue == iter.Current.Key)
455                rowShade.Points.Last().SetValueY(ecdf / totalTargets, (ecdf + missedecdf) / totalTargets);
456              else rowShade.Points.Add(new DataPoint(iter.Current.Key, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
457            }
458            moreMisses = iter.MoveNext();
459            if (!labelPrinted) {
460              var point = row.Points.Last();
461              if (showLabelsInTargetChart)
462                point.Label = row.Name;
463              point.MarkerStyle = MarkerStyle.Cross;
464              point.MarkerBorderWidth = 1;
465              point.MarkerSize = 10;
466              labelPrinted = true;
467            }
468          }
469          ecdf += h.Value;
470          if (row.Points.Count > 0 && row.Points.Last().XValue == h.Key) {
471            row.Points.Last().SetValueY(ecdf / movingTargets);
472            row.Points.Last().Color = Color.FromArgb(255 - (int)Math.Floor(255 * (missedecdf / totalTargets)), colors[colorCount]);
473          } else {
474            var dp = new DataPoint(h.Key, ecdf / movingTargets) {
475              Color = Color.FromArgb(255 - (int)Math.Floor(255 * (missedecdf / totalTargets)), colors[colorCount])
476            };
477            if (showMarkers) {
478              dp.MarkerStyle = MarkerStyle.Circle;
479              dp.MarkerBorderWidth = 1;
480              dp.MarkerSize = 5;
481            }
482            row.Points.Add(dp);
483          }
484          if (missedecdf > 0 && boundShadingCheckBox.Checked) {
485            if (rowShade.Points.Count > 0 && rowShade.Points.Last().XValue == h.Key)
486              rowShade.Points.Last().SetValueY(ecdf / totalTargets, (ecdf + missedecdf) / totalTargets);
487            else rowShade.Points.Add(new DataPoint(h.Key, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
488          }
489        }
490
491        while (moreMisses) {
492          // if there are misses beyond the last hit we extend the shaded area
493          missedecdf += iter.Current.Value;
494          var dp = new DataPoint(iter.Current.Key, ecdf / movingTargets) {
495            Color = Color.FromArgb(255 - (int)Math.Floor(255 * (missedecdf / totalTargets)), colors[colorCount])
496          };
497          if (showMarkers) {
498            dp.MarkerStyle = MarkerStyle.Circle;
499            dp.MarkerBorderWidth = 1;
500            dp.MarkerSize = 5;
501          }
502          row.Points.Add(dp);
503          if (boundShadingCheckBox.Checked) {
504            rowShade.Points.Add(new DataPoint(iter.Current.Key, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
505          }
506          moreMisses = iter.MoveNext();
507
508          if (!labelPrinted && row.Points.Count > 0) {
509            var point = row.Points.Last();
510            if (showLabelsInTargetChart)
511              point.Label = row.Name;
512            point.MarkerStyle = MarkerStyle.Cross;
513            point.MarkerBorderWidth = 1;
514            point.MarkerSize = 10;
515            labelPrinted = true;
516          }
517        }
518
519        if (!labelPrinted) {
520          var point = row.Points.Last();
521          if (showLabelsInTargetChart)
522            point.Label = row.Name;
523          point.MarkerStyle = MarkerStyle.Cross;
524          point.MarkerBorderWidth = 1;
525          point.MarkerSize = 10;
526          rowShade.Points.Add(new DataPoint(point.XValue, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
527          labelPrinted = true;
528        }
529
530        ConfigureSeries(row);
531        targetChart.Series.Add(rowShade);
532        targetChart.Series.Add(row);
533
534        colorCount = (colorCount + 1) % colors.Length;
535        if (colorCount == 0) lineStyleCount = (lineStyleCount + 1) % lineStyles.Length;
536      }
537    }
538
539    private void UpdateTargetChartAxisXBounds(double minEff, double maxEff) {
540      var minZeros = (int)Math.Floor(Math.Log10(minEff));
541      var maxZeros = (int)Math.Floor(Math.Log10(maxEff));
542      var axisMin = (decimal)Math.Pow(10, minZeros);
543      var axisMax = (decimal)Math.Pow(10, maxZeros);
544      if (!targetLogScalingCheckBox.Checked) {
545        var minAdd = (decimal)Math.Pow(10, minZeros - 1) * 2;
546        var maxAdd = (decimal)Math.Pow(10, maxZeros - 1) * 2;
547        while (axisMin + minAdd < (decimal)minEff) axisMin += minAdd;
548        while (axisMax <= (decimal)maxEff) axisMax += maxAdd;
549      } else axisMax = (decimal)Math.Pow(10, (int)Math.Ceiling(Math.Log10(maxEff)));
550      targetChart.ChartAreas[0].AxisX.Minimum = (double)axisMin;
551      targetChart.ChartAreas[0].AxisX.Maximum = (double)axisMax;
552    }
553
554    private IEnumerable<double> GetAbsoluteTargets(ProblemInstance pInstance) {
555      if (!targetsAreRelative) return targets;
556      var maximization = pInstance.IsMaximization();
557      var bestKnown = pInstance.BestKnownQuality;
558      if (double.IsNaN(bestKnown)) throw new ArgumentException("Problem instance does not have a defined best - known quality.");
559      IEnumerable<double> tmp = null;
560      if (bestKnown > 0) {
561        tmp = targets.Select(x => (maximization ? (1 - x) : (1 + x)) * bestKnown);
562      } else if (bestKnown < 0) {
563        tmp = targets.Select(x => (!maximization ? (1 - x) : (1 + x)) * bestKnown);
564      } else {
565        // relative to 0 is impossible
566        tmp = targets;
567      }
568      return tmp;
569    }
570
571    private double[] GetAbsoluteTargetsWorstToBest(ProblemInstance pInstance) {
572      if (double.IsNaN(pInstance.BestKnownQuality)) throw new ArgumentException("Problem instance does not have a defined best-known quality.");
573      var absTargets = GetAbsoluteTargets(pInstance);
574      return (pInstance.IsMaximization()
575        ? absTargets.OrderBy(x => x) : absTargets.OrderByDescending(x => x)).ToArray();
576    }
577
578    private void GenerateDefaultTargets() {
579      targets = new[] { 0.1, 0.095, 0.09, 0.085, 0.08, 0.075, 0.07, 0.065, 0.06, 0.055, 0.05, 0.045, 0.04, 0.035, 0.03, 0.025, 0.02, 0.015, 0.01, 0.005, 0 };
580      SynchronizeTargetTextBox();
581    }
582
583    private Tuple<bool, double> GetEffortToHitTarget(
584        ObservableList<Tuple<double, double>> convergenceGraph,
585        double absTarget, bool maximization) {
586      if (convergenceGraph.Count == 0)
587        throw new ArgumentException("Convergence graph is empty.", "convergenceGraph");
588     
589      var index = convergenceGraph.BinarySearch(Tuple.Create(0.0, absTarget), new TargetComparer(maximization));
590      if (index >= 0) {
591        return Tuple.Create(true, convergenceGraph[index].Item1);
592      } else {
593        index = ~index;
594        if (index >= convergenceGraph.Count)
595          return Tuple.Create(false, convergenceGraph.Last().Item1);
596        return Tuple.Create(true, convergenceGraph[index].Item1);
597      }
598    }
599
600    private void UpdateErtTables(List<AlgorithmInstance> algorithmInstances) {
601      ertTableView.Content = null;
602      var columns = targets.Length + 1;
603      var totalRows = algorithmInstances.Count * algorithmInstances.Max(x => x.GetNumberOfProblemInstances()) + algorithmInstances.Max(x => x.GetNumberOfProblemInstances());
604      var matrix = new StringMatrix(totalRows, columns);
605      var rowNames = new List<string>();
606      matrix.ColumnNames = targets.Select(x => targetsAreRelative ? (100 * x).ToString() + "%" : x.ToString())
607        .Concat(new[] { "#succ" }).ToList();
608      var rowCount = 0;
609
610      var tableName = (string)dataTableComboBox.SelectedItem;
611      if (string.IsNullOrEmpty(tableName)) return;
612     
613      var problems = algorithmInstances.SelectMany(x => x.GetProblemInstances()).Distinct().ToList();
614
615      foreach (var problem in problems) {
616        var max = problem.IsMaximization();
617        rowNames.Add(problem.ToString());
618        var absTargets = GetAbsoluteTargetsWorstToBest(problem);
619        if (targetsAreRelative) {
620          // print out the absolute target values
621          for (var i = 0; i < absTargets.Length; i++) {
622            matrix[rowCount, i] = absTargets[i].ToString("##,0.0", CultureInfo.CurrentCulture.NumberFormat);
623          }
624        }
625        rowCount++;
626
627        foreach (var alg in algorithmInstances) {
628          rowNames.Add(alg.Name);
629          var runs = alg.GetRuns(problem).ToList();
630          if (runs.Count == 0) {
631            matrix[rowCount, columns - 1] = "N/A";
632            rowCount++;
633            continue;
634          }
635          var result = default(ErtCalculationResult);
636          for (var i = 0; i < absTargets.Length; i++) {
637            result = ExpectedRuntimeHelper.CalculateErt(runs, tableName, absTargets[i], max);
638            matrix[rowCount, i] = result.ToString();
639          }
640          matrix[rowCount, columns - 1] = targets.Length > 0 ? result.SuccessfulRuns + "/" + result.TotalRuns : "-";
641          rowCount++;
642        }
643      }
644      matrix.RowNames = rowNames;
645      ertTableView.Content = matrix;
646      ertTableView.DataGridView.AutoResizeColumns(DataGridViewAutoSizeColumnsMode.AllCells);
647    }
648    #endregion
649
650    #region Performance analysis by (multiple) budget(s)
651    private void UpdateResultsByCost() {
652      // necessary to reset log scale -> empty chart cannot use log scaling
653      byCostDataTable.VisualProperties.XAxisLogScale = false;
654      byCostDataTable.Rows.Clear();
655
656      var table = (string)dataTableComboBox.SelectedItem;
657      if (string.IsNullOrEmpty(table)) return;
658
659      if (budgets == null) GenerateDefaultBudgets(table);
660
661      var algInstances = GroupRuns().ToList();
662      if (algInstances.Count == 0) return;
663
664      var colorCount = 0;
665      var lineStyleCount = 0;
666     
667      foreach (var alg in algInstances) {
668        var hits = new Dictionary<string, SortedList<double, double>>();
669
670        foreach (var problem in alg.GetProblemInstances()) {
671          foreach (var run in alg.GetRuns(problem)) {
672            var resultsTable = (IndexedDataTable<double>)run.Results[table];
673
674            CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), problem, alg.Name);
675          }
676        }
677
678        foreach (var list in hits) {
679          var row = new IndexedDataRow<double>(list.Key) {
680            VisualProperties = {
681              ChartType = DataRowVisualProperties.DataRowChartType.StepLine,
682              LineWidth = 2,
683              Color = colors[colorCount],
684              LineStyle = hlLineStyles[lineStyleCount],
685              StartIndexZero = false
686            }
687          };
688
689          var total = 0.0;
690          var count = list.Value.Count;
691          foreach (var h in list.Value) {
692            total += h.Value;
693            row.Values.Add(Tuple.Create(h.Key, total / (double)count));
694          }
695
696          byCostDataTable.Rows.Add(row);
697        }
698        colorCount = (colorCount + 1) % colors.Length;
699        if (colorCount == 0) lineStyleCount = (lineStyleCount + 1) % lineStyles.Length;
700      }
701
702      byCostDataTable.VisualProperties.XAxisTitle = "Targets to Best-Known Ratio";
703      byCostDataTable.VisualProperties.XAxisLogScale = byCostDataTable.Rows.Count > 0 && budgetLogScalingCheckBox.Checked;
704    }
705
706    private void GenerateDefaultBudgets(string table) {
707      var runs = Content;
708      var min = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Min()).Min();
709      var max = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Max()).Max();
710      var points = 3;
711      budgets = Enumerable.Range(1, points).Select(x => min + (x / (double)points) * (max - min)).ToArray();
712      suppressBudgetsEvents = true;
713      budgetsTextBox.Text = string.Join(" ; ", budgets);
714      suppressBudgetsEvents = false;
715    }
716
717    private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, ProblemInstance problem, string groupName) {
718      var max = problem.IsMaximization();
719      var prevIndex = 0;
720      foreach (var b in budgets) {
721        var key = groupName + "-" + b;
722        var index = row.Values.BinarySearch(prevIndex, row.Values.Count - prevIndex, Tuple.Create(b, 0.0), new CostComparer());
723        if (index < 0) {
724          index = ~index;
725          if (index >= row.Values.Count) break; // the run wasn't long enough to use up budget b (or any subsequent larger one)
726        }
727        if (!hits.ContainsKey(key)) hits.Add(key, new SortedList<double, double>());
728        var v = row.Values[index];
729        var relTgt = CalculateRelativeDifference(max, problem.BestKnownQuality, v.Item2) + 1;
730        if (hits[key].ContainsKey(relTgt))
731          hits[key][relTgt]++;
732        else hits[key][relTgt] = 1.0;
733        prevIndex = index;
734      }
735    }
736    #endregion
737
738    private void UpdateCaption() {
739      Caption = Content != null ? Content.OptimizerName + " RLD View" : ViewAttribute.GetViewName(GetType());
740    }
741
742    private void SynchronizeTargetTextBox() {
743      if (InvokeRequired) Invoke((Action)SynchronizeTargetTextBox);
744      else {
745        suppressTargetsEvents = true;
746        try {
747          if (targetsAreRelative)
748            targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100)) + "%";
749          else targetsTextBox.Text = string.Join(" ; ", targets);
750        } finally { suppressTargetsEvents = false; }
751      }
752    }
753
754    private void groupComboBox_SelectedIndexChanged(object sender, EventArgs e) {
755      UpdateRuns();
756      SetEnabledStateOfControls();
757    }
758    private void problemComboBox_SelectedIndexChanged(object sender, EventArgs e) {
759      UpdateRuns();
760      SetEnabledStateOfControls();
761    }
762    private void dataTableComboBox_SelectedIndexChanged(object sender, EventArgs e) {
763      if (dataTableComboBox.SelectedIndex >= 0)
764        GenerateDefaultBudgets((string)dataTableComboBox.SelectedItem);
765      UpdateBestKnownQualities();
766      UpdateRuns();
767      SetEnabledStateOfControls();
768    }
769
770    private void logScalingCheckBox_CheckedChanged(object sender, EventArgs e) {
771      UpdateResultsByTarget();
772      byCostDataTable.VisualProperties.XAxisLogScale = byCostDataTable.Rows.Count > 0 && budgetLogScalingCheckBox.Checked;
773    }
774
775    private void boundShadingCheckBox_CheckedChanged(object sender, EventArgs e) {
776      UpdateResultsByTarget();
777    }
778
779    #region Event handlers for target analysis
780    private bool suppressTargetsEvents;
781    private void targetsTextBox_Validating(object sender, CancelEventArgs e) {
782      if (suppressTargetsEvents) return;
783      var targetStrings = targetsTextBox.Text.Split(new[] { '%', ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
784      var targetList = new List<decimal>();
785      foreach (var ts in targetStrings) {
786        decimal t;
787        if (!decimal.TryParse(ts, out t)) {
788          errorProvider.SetError(targetsTextBox, "Not all targets can be parsed: " + ts);
789          e.Cancel = true;
790          return;
791        }
792        if (targetsAreRelative)
793          targetList.Add(t / 100);
794        else targetList.Add(t);
795      }
796      if (targetList.Count == 0) {
797        errorProvider.SetError(targetsTextBox, "Give at least one target value!");
798        e.Cancel = true;
799        return;
800      }
801      e.Cancel = false;
802      errorProvider.SetError(targetsTextBox, null);
803      targets = targetsAreRelative ? targetList.Select(x => (double)x).OrderByDescending(x => x).ToArray() : targetList.Select(x => (double)x).ToArray();
804
805      SynchronizeTargetTextBox();
806      UpdateResultsByTarget();
807      SetEnabledStateOfControls();
808    }
809
810    private void aggregateTargetsCheckBox_CheckedChanged(object sender, EventArgs e) {
811      SuspendRepaint();
812      try {
813        UpdateResultsByTarget();
814      } finally { ResumeRepaint(true); }
815    }
816
817    private void relativeOrAbsoluteComboBox_SelectedIndexChanged(object sender, EventArgs e) {
818      if (suppressUpdates) return;
819      var pd = (ProblemInstance)problemComboBox.SelectedItem;
820      if (!double.IsNaN(pd.BestKnownQuality)) {
821        var max = pd.IsMaximization();
822        if (targetsAreRelative) targets = GetAbsoluteTargets(pd).ToArray();
823        else {
824          // Rounding to 5 digits since it's certainly appropriate for this application
825          if (pd.BestKnownQuality > 0) {
826            targets = targets.Select(x => Math.Round(max ? 1.0 - (x / pd.BestKnownQuality) : (x / pd.BestKnownQuality) - 1.0, 5)).ToArray();
827          } else if (pd.BestKnownQuality < 0) {
828            targets = targets.Select(x => Math.Round(!max ? 1.0 - (x / pd.BestKnownQuality) : (x / pd.BestKnownQuality) - 1.0, 5)).ToArray();
829          }
830        }
831      }
832      targetsAreRelative = (string)relativeOrAbsoluteComboBox.SelectedItem == "relative";
833      SynchronizeTargetTextBox();
834
835      try {
836        SuspendRepaint();
837        UpdateResultsByTarget();
838      } finally { ResumeRepaint(true); }
839    }
840
841    private void generateTargetsButton_Click(object sender, EventArgs e) {
842      if (targets == null) return;
843      decimal max = 10, min = 0, count = 10;
844      max = (decimal)targets.Max();
845      min = (decimal)targets.Min();
846      count = targets.Length - 1;
847      if (targetsAreRelative) {
848        max *= 100;
849        min *= 100;
850      }
851      using (var dialog = new DefineArithmeticProgressionDialog(false, min, max, (max - min) / count)) {
852        if (dialog.ShowDialog() == DialogResult.OK) {
853          if (dialog.Values.Any()) {
854            targets = targetsAreRelative
855              ? dialog.Values.OrderByDescending(x => x).Select(x => (double)x / 100.0).ToArray()
856              : dialog.Values.Select(x => (double)x).ToArray();
857
858            SynchronizeTargetTextBox();
859            UpdateResultsByTarget();
860            SetEnabledStateOfControls();
861          }
862        }
863      }
864    }
865
866    private void addTargetsAsResultButton_Click(object sender, EventArgs e) {
867      var table = (string)dataTableComboBox.SelectedItem;
868      if (string.IsNullOrEmpty(table)) return;
869     
870      foreach (var run in Content) {
871        if (!run.Results.ContainsKey(table)) continue;
872        var resultsTable = (IndexedDataTable<double>)run.Results[table];
873        var values = resultsTable.Rows.First().Values;
874        var pd = new ProblemInstance(run);
875        pd = problems.Single(x => x.Equals(pd));
876        var max = pd.IsMaximization();
877        var absTargets = GetAbsoluteTargetsWorstToBest(pd);
878
879        var prevIndex = 0;
880        for (var i = 0; i < absTargets.Length; i++) {
881          var absTarget = absTargets[i];
882          var index = values.BinarySearch(prevIndex, values.Count - prevIndex, Tuple.Create(0.0, absTarget), new TargetComparer(max));
883          if (index < 0) {
884            index = ~index;
885            if (index >= values.Count) break; // the target (and subsequent ones) wasn't achieved
886          }
887          var target = targetsAreRelative ? (targets[i] * 100) : absTarget;
888          run.Results[table + (targetsAreRelative ? ".RelTarget " : ".AbsTarget ") + target + (targetsAreRelative ? "%" : string.Empty)] = new DoubleValue(values[index].Item1);
889          prevIndex = index;
890        }
891      }
892    }
893
894    private void markerCheckBox_CheckedChanged(object sender, EventArgs e) {
895      SuspendRepaint();
896      try {
897        UpdateResultsByTarget();
898      } finally { ResumeRepaint(true); }
899    }
900
901    private void showLabelsCheckBox_CheckedChanged(object sender, EventArgs e) {
902      showLabelsInTargetChart = showLabelsCheckBox.Checked;
903      SuspendRepaint();
904      try {
905        UpdateResultsByTarget();
906      } finally { ResumeRepaint(true); }
907    }
908    #endregion
909
910    #region Event handlers for cost analysis
911    private bool suppressBudgetsEvents;
912    private void budgetsTextBox_Validating(object sender, CancelEventArgs e) {
913      if (suppressBudgetsEvents) return;
914      var budgetStrings = budgetsTextBox.Text.Split(new[] { ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
915      var budgetList = new List<double>();
916      foreach (var ts in budgetStrings) {
917        double b;
918        if (!double.TryParse(ts, out b)) {
919          errorProvider.SetError(budgetsTextBox, "Not all budgets can be parsed: " + ts);
920          e.Cancel = true;
921          return;
922        }
923        budgetList.Add(b);
924      }
925      if (budgetList.Count == 0) {
926        errorProvider.SetError(budgetsTextBox, "Give at least one budget value!");
927        e.Cancel = true;
928        return;
929      }
930      e.Cancel = false;
931      errorProvider.SetError(budgetsTextBox, null);
932      budgets = budgetList.OrderBy(x => x).ToArray();
933      try {
934        suppressBudgetsEvents = true;
935        budgetsTextBox.Text = string.Join(" ; ", budgets);
936      } finally { suppressBudgetsEvents = false; }
937      UpdateResultsByCost();
938      SetEnabledStateOfControls();
939    }
940
941    private void generateBudgetsButton_Click(object sender, EventArgs e) {
942      decimal max = 1, min = 0, count = 10;
943      if (budgets != null) {
944        max = (decimal)budgets.Max();
945        min = (decimal)budgets.Min();
946        count = budgets.Length;
947      } else if (Content.Count > 0 && dataTableComboBox.SelectedIndex >= 0) {
948        var table = (string)dataTableComboBox.SelectedItem;
949        min = (decimal)Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Min(y => y.Item1)).Min();
950        max = (decimal)Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Max(y => y.Item1)).Max();
951        count = 3;
952      }
953      using (var dialog = new DefineArithmeticProgressionDialog(false, min, max, (max - min) / count)) {
954        if (dialog.ShowDialog() == DialogResult.OK) {
955          if (dialog.Values.Any()) {
956            budgets = dialog.Values.OrderBy(x => x).Select(x => (double)x).ToArray();
957
958            try {
959              suppressBudgetsEvents = true;
960              budgetsTextBox.Text = string.Join(" ; ", budgets);
961            } finally { suppressBudgetsEvents = false; }
962
963            UpdateResultsByCost();
964            SetEnabledStateOfControls();
965          }
966        }
967      }
968    }
969
970    private void addBudgetsAsResultButton_Click(object sender, EventArgs e) {
971      var table = (string)dataTableComboBox.SelectedItem;
972
973      foreach (var run in Content) {
974        if (!run.Results.ContainsKey(table)) continue;
975        var resultsTable = (IndexedDataTable<double>)run.Results[table];
976        var values = resultsTable.Rows.First().Values;
977        var pd = new ProblemInstance(run);
978        pd = problems.Single(x => x.Equals(pd));
979
980        var prevIndex = 0;
981        foreach (var b in budgets) {
982          var index = values.BinarySearch(prevIndex, values.Count - prevIndex, Tuple.Create(b, 0.0), new CostComparer());
983          if (index < 0) {
984            index = ~index;
985            if (index >= values.Count) break; // the run wasn't long enough to use up budget b (or any subsequent larger one)
986          }
987          var v = values[index];
988          var tgt = targetsAreRelative ? CalculateRelativeDifference(pd.IsMaximization(), pd.BestKnownQuality, v.Item2) : v.Item2;
989          run.Results[table + (targetsAreRelative ? ".CostForRelTarget " : ".CostForAbsTarget ") + b] = new DoubleValue(tgt);
990          prevIndex = index;
991        }
992      }
993    }
994    #endregion
995
996    #region Helpers
997    private double CalculateRelativeDifference(bool maximization, double bestKnown, double fit) {
998      if (bestKnown == 0) {
999        // no relative difference with respect to bestKnown possible
1000        return maximization ? -fit : fit;
1001      }
1002      var absDiff = (fit - bestKnown);
1003      var relDiff = absDiff / bestKnown;
1004      if (maximization) {
1005        return bestKnown > 0 ? -relDiff : relDiff;
1006      } else {
1007        return bestKnown > 0 ? relDiff : -relDiff;
1008      }
1009    }
1010
1011    private Dictionary<ProblemInstance, double> CalculateBestTargetPerProblemInstance(string table) {
1012      if (table == null) table = string.Empty;
1013      return (from r in Content
1014              where r.Visible
1015              let pd = new ProblemInstance(r)
1016              let target = r.Parameters.ContainsKey("BestKnownQuality")
1017                           && r.Parameters["BestKnownQuality"] is DoubleValue
1018                ? ((DoubleValue)r.Parameters["BestKnownQuality"]).Value
1019                : (r.Results.ContainsKey(table) && r.Results[table] is IndexedDataTable<double>
1020                  ? ((IndexedDataTable<double>)r.Results[table]).Rows.First().Values.Last().Item2
1021                  : double.NaN)
1022              group target by pd into g
1023              select new { Problem = g.Key, Target = g.Any(x => !double.IsNaN(x)) ? (g.Key.IsMaximization() ? g.Max() : g.Where(x => !double.IsNaN(x)).Min()) : double.NaN })
1024        .ToDictionary(x => x.Problem, x => x.Target);
1025    }
1026
1027    private void UpdateRuns() {
1028      if (InvokeRequired) {
1029        Invoke((Action)UpdateRuns);
1030        return;
1031      }
1032      SuspendRepaint();
1033      try {
1034        UpdateResultsByTarget();
1035        UpdateResultsByCost();
1036      } finally { ResumeRepaint(true); }
1037    }
1038
1039    private void UpdateBestKnownQualities() {
1040      var table = (string)dataTableComboBox.SelectedItem;
1041      if (string.IsNullOrEmpty(table)) return;
1042
1043      var targetsPerProblem = CalculateBestTargetPerProblemInstance(table);
1044      foreach (var pd in problems) {
1045        double bkq;
1046        if (targetsPerProblem.TryGetValue(pd, out bkq))
1047          pd.BestKnownQuality = bkq;
1048        else pd.BestKnownQuality = double.NaN;
1049      }
1050    }
1051    #endregion
1052
1053    private void ConfigureSeries(Series series) {
1054      series.SmartLabelStyle.Enabled = showLabelsInTargetChart;
1055      series.SmartLabelStyle.AllowOutsidePlotArea = LabelOutsidePlotAreaStyle.No;
1056      series.SmartLabelStyle.CalloutLineAnchorCapStyle = LineAnchorCapStyle.None;
1057      series.SmartLabelStyle.CalloutLineColor = series.Color;
1058      series.SmartLabelStyle.CalloutLineWidth = 2;
1059      series.SmartLabelStyle.CalloutStyle = LabelCalloutStyle.Underlined;
1060      series.SmartLabelStyle.IsOverlappedHidden = false;
1061      series.SmartLabelStyle.MaxMovingDistance = 200;
1062      series.ToolTip = series.LegendText + " X = #VALX, Y = #VALY";
1063    }
1064
1065    private void chart_MouseDown(object sender, MouseEventArgs e) {
1066      HitTestResult result = targetChart.HitTest(e.X, e.Y);
1067      if (result.ChartElementType == ChartElementType.LegendItem) {
1068        ToggleTargetChartSeriesVisible(result.Series);
1069      }
1070    }
1071    private void chart_MouseMove(object sender, MouseEventArgs e) {
1072      HitTestResult result = targetChart.HitTest(e.X, e.Y);
1073      if (result.ChartElementType == ChartElementType.LegendItem)
1074        this.Cursor = Cursors.Hand;
1075      else
1076        this.Cursor = Cursors.Default;
1077    }
1078    private void chart_CustomizeLegend(object sender, CustomizeLegendEventArgs e) {
1079      foreach (LegendItem legendItem in e.LegendItems) {
1080        var series = targetChart.Series[legendItem.SeriesName];
1081        if (series != null) {
1082          bool seriesIsInvisible = invisibleTargetSeries.Any(x => x.Name == series.Name);
1083          foreach (LegendCell cell in legendItem.Cells) {
1084            cell.ForeColor = seriesIsInvisible ? Color.Gray : Color.Black;
1085          }
1086        }
1087      }
1088    }
1089
1090    private void ToggleTargetChartSeriesVisible(Series series) {
1091      var indexList = invisibleTargetSeries.FindIndex(x => x.Name == series.Name);
1092      var indexChart = targetChart.Series.IndexOf(series);
1093      if (targetChart.Series.Count == 1) targetChart.ChartAreas[0].AxisX.IsLogarithmic = false;
1094      targetChart.Series.RemoveAt(indexChart);
1095      var s = indexList >= 0 ? invisibleTargetSeries[indexList] : new Series(series.Name) {
1096        Color = series.Color,
1097        ChartType = series.ChartType,
1098        BorderWidth = series.BorderWidth,
1099        BorderDashStyle = series.BorderDashStyle
1100      };
1101      if (indexList < 0) {
1102        // hide
1103        invisibleTargetSeries.Add(series);
1104        var shadeSeries = targetChart.Series.FirstOrDefault(x => x.Name == series.Name + "-range");
1105        if (shadeSeries != null) {
1106          if (targetChart.Series.Count == 1) targetChart.ChartAreas[0].AxisX.IsLogarithmic = false;
1107          targetChart.Series.Remove(shadeSeries);
1108          invisibleTargetSeries.Add(shadeSeries);
1109          indexChart--;
1110        }
1111      } else {
1112        // show
1113        invisibleTargetSeries.RemoveAt(indexList);
1114        var shadeSeries = invisibleTargetSeries.FirstOrDefault(x => x.Name == series.Name + "-range");
1115        if (shadeSeries != null) {
1116          invisibleTargetSeries.Remove(shadeSeries);
1117          InsertOrAddSeries(indexChart, shadeSeries);
1118          indexChart++;
1119        }
1120      }
1121      InsertOrAddSeries(indexChart, s);
1122      targetChart.ChartAreas[0].AxisX.IsLogarithmic = CanDisplayLogarithmic();
1123    }
1124
1125    private bool CanDisplayLogarithmic() {
1126      return targetLogScalingCheckBox.Checked
1127        && targetChart.Series.Count > 0 // must have a series
1128        && targetChart.Series.Any(x => x.Points.Count > 0) // at least one series must have points
1129        && targetChart.Series.All(s => s.Points.All(p => p.XValue > 0)); // all points must be positive
1130    }
1131
1132    private void InsertOrAddSeries(int index, Series s) {
1133      if (targetChart.Series.Count <= index)
1134        targetChart.Series.Add(s);
1135      else targetChart.Series.Insert(index, s);
1136    }
1137
1138    private class AlgorithmInstance : INotifyPropertyChanged {
1139      private string name;
1140      public string Name {
1141        get { return name; }
1142        set {
1143          if (name == value) return;
1144          name = value;
1145          OnPropertyChanged("Name");
1146        }
1147      }
1148
1149      private Dictionary<ProblemInstance, List<IRun>> performanceData;
1150
1151      public int GetNumberOfProblemInstances() {
1152        return performanceData.Count;
1153      }
1154
1155      public IEnumerable<ProblemInstance> GetProblemInstances() {
1156        return performanceData.Keys;
1157      }
1158
1159      public int GetNumberOfRuns(ProblemInstance p) {
1160        if (p == ProblemInstance.MatchAll) return performanceData.Select(x => x.Value.Count).Sum();
1161        List<IRun> runs;
1162        if (performanceData.TryGetValue(p, out runs))
1163          return runs.Count;
1164
1165        return 0;
1166      }
1167
1168      public IEnumerable<IRun> GetRuns(ProblemInstance p) {
1169        if (p == ProblemInstance.MatchAll) return performanceData.SelectMany(x => x.Value);
1170        List<IRun> runs;
1171        if (performanceData.TryGetValue(p, out runs))
1172          return runs;
1173
1174        return Enumerable.Empty<IRun>();
1175      }
1176
1177      public AlgorithmInstance(string name, IEnumerable<IRun> runs, IEnumerable<ProblemInstance> problems) {
1178        this.name = name;
1179
1180        var pDict = problems.ToDictionary(r => r, _ => new List<IRun>());
1181        foreach (var y in runs) {
1182          var pd = new ProblemInstance(y);
1183          List<IRun> l;
1184          if (pDict.TryGetValue(pd, out l)) l.Add(y);
1185        }
1186        performanceData = pDict.Where(x => x.Value.Count > 0).ToDictionary(x => x.Key, x => x.Value);
1187      }
1188
1189      public override bool Equals(object obj) {
1190        var other = obj as AlgorithmInstance;
1191        if (other == null) return false;
1192        return name == other.name;
1193      }
1194
1195      public override int GetHashCode() {
1196        return name.GetHashCode();
1197      }
1198
1199      public event PropertyChangedEventHandler PropertyChanged;
1200      protected virtual void OnPropertyChanged(string propertyName = null) {
1201        var handler = PropertyChanged;
1202        if (handler != null) handler(this, new PropertyChangedEventArgs(propertyName));
1203      }
1204    }
1205
1206    private class ProblemInstance : INotifyPropertyChanged {
1207      private readonly bool matchAll;
1208      public static readonly ProblemInstance MatchAll = new ProblemInstance() {
1209        ProblemName = "All with Best-Known"
1210      };
1211
1212      private ProblemInstance() {
1213        ProblemType = string.Empty;
1214        ProblemName = string.Empty;
1215        Evaluator = string.Empty;
1216        Maximization = string.Empty;
1217        DisplayProblemType = false;
1218        DisplayProblemName = false;
1219        DisplayEvaluator = false;
1220        DisplayMaximization = false;
1221        matchAll = true;
1222        BestKnownQuality = double.NaN;
1223      }
1224
1225      public ProblemInstance(IRun run) {
1226        ProblemType = GetStringValueOrEmpty(run, "Problem Type");
1227        ProblemName = GetStringValueOrEmpty(run, "Problem Name");
1228        Evaluator = GetStringValueOrEmpty(run, "Evaluator");
1229        Maximization = GetMaximizationValueOrEmpty(run, "Maximization");
1230        DisplayProblemType = !string.IsNullOrEmpty(ProblemType);
1231        DisplayProblemName = !string.IsNullOrEmpty(ProblemName);
1232        DisplayEvaluator = !string.IsNullOrEmpty(Evaluator);
1233        DisplayMaximization = !string.IsNullOrEmpty(Maximization);
1234        matchAll = false;
1235        BestKnownQuality = GetDoubleValueOrNaN(run, "BestKnownQuality");
1236      }
1237
1238      private bool displayProblemType;
1239      public bool DisplayProblemType {
1240        get { return displayProblemType; }
1241        set {
1242          if (displayProblemType == value) return;
1243          displayProblemType = value;
1244          OnPropertyChanged("DisplayProblemType");
1245        }
1246      }
1247      private string problemType;
1248      public string ProblemType {
1249        get { return problemType; }
1250        set {
1251          if (problemType == value) return;
1252          problemType = value;
1253          OnPropertyChanged("ProblemType");
1254        }
1255      }
1256      private bool displayProblemName;
1257      public bool DisplayProblemName {
1258        get { return displayProblemName; }
1259        set {
1260          if (displayProblemName == value) return;
1261          displayProblemName = value;
1262          OnPropertyChanged("DisplayProblemName");
1263        }
1264      }
1265      private string problemName;
1266      public string ProblemName {
1267        get { return problemName; }
1268        set {
1269          if (problemName == value) return;
1270          problemName = value;
1271          OnPropertyChanged("ProblemName");
1272        }
1273      }
1274      private bool displayEvaluator;
1275      public bool DisplayEvaluator {
1276        get { return displayEvaluator; }
1277        set {
1278          if (displayEvaluator == value) return;
1279          displayEvaluator = value;
1280          OnPropertyChanged("DisplayEvaluator");
1281        }
1282      }
1283      private string evaluator;
1284      public string Evaluator {
1285        get { return evaluator; }
1286        set {
1287          if (evaluator == value) return;
1288          evaluator = value;
1289          OnPropertyChanged("Evaluator");
1290        }
1291      }
1292      private bool displayMaximization;
1293      public bool DisplayMaximization {
1294        get { return displayMaximization; }
1295        set {
1296          if (displayMaximization == value) return;
1297          displayMaximization = value;
1298          OnPropertyChanged("DisplayMaximization");
1299        }
1300      }
1301      private string maximization;
1302      public string Maximization {
1303        get { return maximization; }
1304        set {
1305          if (maximization == value) return;
1306          maximization = value;
1307          OnPropertyChanged("Maximization");
1308        }
1309      }
1310      private double bestKnownQuality;
1311      public double BestKnownQuality {
1312        get { return bestKnownQuality; }
1313        set {
1314          if (bestKnownQuality == value) return;
1315          bestKnownQuality = value;
1316          OnPropertyChanged("BestKnownQuality");
1317        }
1318      }
1319
1320      public bool IsMaximization() {
1321        return Maximization == "MAX";
1322      }
1323
1324      public bool Match(IRun run) {
1325        return matchAll ||
1326               GetStringValueOrEmpty(run, "Problem Type") == ProblemType
1327               && GetStringValueOrEmpty(run, "Problem Name") == ProblemName
1328               && GetStringValueOrEmpty(run, "Evaluator") == Evaluator
1329               && GetMaximizationValueOrEmpty(run, "Maximization") == Maximization;
1330      }
1331
1332      private double GetDoubleValueOrNaN(IRun run, string key) {
1333        IItem param;
1334        if (run.Parameters.TryGetValue(key, out param)) {
1335          var dv = param as DoubleValue;
1336          return dv != null ? dv.Value : double.NaN;
1337        }
1338        return double.NaN;
1339      }
1340
1341      private string GetStringValueOrEmpty(IRun run, string key) {
1342        IItem param;
1343        if (run.Parameters.TryGetValue(key, out param)) {
1344          var sv = param as StringValue;
1345          return sv != null ? sv.Value : string.Empty;
1346        }
1347        return string.Empty;
1348      }
1349
1350      private string GetMaximizationValueOrEmpty(IRun run, string key) {
1351        IItem param;
1352        if (run.Parameters.TryGetValue(key, out param)) {
1353          var bv = param as BoolValue;
1354          return bv != null ? (bv.Value ? "MAX" : "MIN") : string.Empty;
1355        }
1356        return string.Empty;
1357      }
1358
1359      public override bool Equals(object obj) {
1360        var other = obj as ProblemInstance;
1361        if (other == null) return false;
1362        return ProblemType == other.ProblemType
1363               && ProblemName == other.ProblemName
1364               && Evaluator == other.Evaluator
1365               && Maximization == other.Maximization;
1366      }
1367
1368      public override int GetHashCode() {
1369        return ProblemType.GetHashCode() ^ ProblemName.GetHashCode() ^ Evaluator.GetHashCode() ^ Maximization.GetHashCode();
1370      }
1371
1372      public override string ToString() {
1373        return string.Join("  --  ", new[] {
1374          (DisplayProblemType ? ProblemType : string.Empty),
1375          (DisplayProblemName ? ProblemName : string.Empty),
1376          (DisplayEvaluator ? Evaluator : string.Empty),
1377          (DisplayMaximization ? Maximization : string.Empty),
1378          !double.IsNaN(BestKnownQuality) ? BestKnownQuality.ToString(CultureInfo.CurrentCulture.NumberFormat) : string.Empty }.Where(x => !string.IsNullOrEmpty(x)));
1379      }
1380
1381      public event PropertyChangedEventHandler PropertyChanged;
1382      protected virtual void OnPropertyChanged(string propertyName = null) {
1383        var handler = PropertyChanged;
1384        if (handler != null) handler(this, new PropertyChangedEventArgs(propertyName));
1385      }
1386    }
1387
1388    private class CostComparer : Comparer<Tuple<double, double>> {
1389      public override int Compare(Tuple<double, double> x, Tuple<double, double> y) {
1390        return x.Item1.CompareTo(y.Item1);
1391      }
1392    }
1393
1394    private class TargetComparer : Comparer<Tuple<double, double>> {
1395      public bool Maximization { get; private set; }
1396      public TargetComparer(bool maximization) {
1397        Maximization = maximization;
1398      }
1399
1400      public override int Compare(Tuple<double, double> x, Tuple<double, double> y) {
1401        return Maximization ? x.Item2.CompareTo(y.Item2) : y.Item2.CompareTo(x.Item2);
1402      }
1403    }
1404  }
1405}
Note: See TracBrowser for help on using the repository browser.