source: branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs @ 14035

Last change on this file since 14035 was 14035, checked in by abeham, 4 years ago

#2431:

  • Fixed bug that occurs when runs of unequal length are aggregated over multiple problem instances
File size: 42.2 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using HeuristicLab.Analysis;
23using HeuristicLab.Collections;
24using HeuristicLab.Core.Views;
25using HeuristicLab.Data;
26using HeuristicLab.MainForm;
27using HeuristicLab.MainForm.WindowsForms;
28using System;
29using System.Collections.Generic;
30using System.ComponentModel;
31using System.Drawing;
32using System.Globalization;
33using System.Linq;
34using System.Windows.Forms;
35
36namespace HeuristicLab.Optimization.Views {
37  [View("Run-length Distribution View")]
38  [Content(typeof(RunCollection), false)]
39  public partial class RunCollectionRLDView : ItemView {
40    private const string AllRuns = "All Runs";
41
42    private static readonly Color[] colors = new[] {
43      Color.FromArgb(0x40, 0x6A, 0xB7),
44      Color.FromArgb(0xB1, 0x6D, 0x01),
45      Color.FromArgb(0x4E, 0x8A, 0x06),
46      Color.FromArgb(0x75, 0x50, 0x7B),
47      Color.FromArgb(0x72, 0x9F, 0xCF),
48      Color.FromArgb(0xA4, 0x00, 0x00),
49      Color.FromArgb(0xAD, 0x7F, 0xA8),
50      Color.FromArgb(0x29, 0x50, 0xCF),
51      Color.FromArgb(0x90, 0xB0, 0x60),
52      Color.FromArgb(0xF5, 0x89, 0x30),
53      Color.FromArgb(0x55, 0x57, 0x53),
54      Color.FromArgb(0xEF, 0x59, 0x59),
55      Color.FromArgb(0xED, 0xD4, 0x30),
56      Color.FromArgb(0x63, 0xC2, 0x16),
57    };
58    private static readonly DataRowVisualProperties.DataRowLineStyle[] lineStyles = new[] {
59      DataRowVisualProperties.DataRowLineStyle.Solid,
60      DataRowVisualProperties.DataRowLineStyle.Dash,
61      DataRowVisualProperties.DataRowLineStyle.DashDot,
62      DataRowVisualProperties.DataRowLineStyle.Dot
63    };
64
65    public new RunCollection Content {
66      get { return (RunCollection)base.Content; }
67      set { base.Content = value; }
68    }
69
70    private double[] targets;
71    private double[] budgets;
72
73    private bool suppressUpdates;
74    private readonly IndexedDataTable<double> byTargetDataTable;
75    public IndexedDataTable<double> ByTargetDataTable {
76      get { return byTargetDataTable; }
77    }
78    private readonly IndexedDataTable<double> byCostDataTable;
79    public IndexedDataTable<double> ByCostDataTable {
80      get { return byCostDataTable; }
81    }
82
83    public RunCollectionRLDView() {
84      InitializeComponent();
85      byTargetDataTable = new IndexedDataTable<double>("ECDF by Target", "A data table containing the ECDF of each of a number of groups.") {
86        VisualProperties = {
87          YAxisTitle = "Proportion of reached targets",
88          YAxisMinimumFixedValue = 0,
89          YAxisMinimumAuto = false,
90          YAxisMaximumFixedValue = 1,
91          YAxisMaximumAuto = false
92        }
93      };
94      byTargetViewHost.Content = byTargetDataTable;
95      byCostDataTable = new IndexedDataTable<double>("ECDF by Cost", "A data table containing the ECDF of each of a number of groups.") {
96        VisualProperties = {
97          YAxisTitle = "Proportion of unused budgets",
98          YAxisMinimumFixedValue = 0,
99          YAxisMinimumAuto = false,
100          YAxisMaximumFixedValue = 1,
101          YAxisMaximumAuto = false
102        }
103      };
104      byCostViewHost.Content = byCostDataTable;
105      suppressUpdates = false;
106    }
107
108    #region Content events
109    protected override void RegisterContentEvents() {
110      base.RegisterContentEvents();
111      Content.ItemsAdded += Content_ItemsAdded;
112      Content.ItemsRemoved += Content_ItemsRemoved;
113      Content.CollectionReset += Content_CollectionReset;
114      Content.UpdateOfRunsInProgressChanged += Content_UpdateOfRunsInProgressChanged;
115      Content.OptimizerNameChanged += Content_AlgorithmNameChanged;
116    }
117    protected override void DeregisterContentEvents() {
118      Content.ItemsAdded -= Content_ItemsAdded;
119      Content.ItemsRemoved -= Content_ItemsRemoved;
120      Content.CollectionReset -= Content_CollectionReset;
121      Content.UpdateOfRunsInProgressChanged -= Content_UpdateOfRunsInProgressChanged;
122      Content.OptimizerNameChanged -= Content_AlgorithmNameChanged;
123      base.DeregisterContentEvents();
124    }
125
126    private void Content_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IRun> e) {
127      if (suppressUpdates) return;
128      if (InvokeRequired) {
129        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_ItemsAdded), sender, e);
130        return;
131      }
132      UpdateGroupAndProblemComboBox();
133      UpdateDataTableComboBox();
134      foreach (var run in e.Items)
135        RegisterRunEvents(run);
136    }
137    private void Content_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IRun> e) {
138      if (suppressUpdates) return;
139      if (InvokeRequired) {
140        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_ItemsRemoved), sender, e);
141        return;
142      }
143      UpdateGroupAndProblemComboBox();
144      UpdateDataTableComboBox();
145      foreach (var run in e.Items)
146        DeregisterRunEvents(run);
147    }
148    private void Content_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
149      if (suppressUpdates) return;
150      if (InvokeRequired) {
151        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset), sender, e);
152        return;
153      }
154      UpdateGroupAndProblemComboBox();
155      UpdateDataTableComboBox();
156      foreach (var run in e.OldItems)
157        DeregisterRunEvents(run);
158    }
159    private void Content_AlgorithmNameChanged(object sender, EventArgs e) {
160      if (InvokeRequired)
161        Invoke(new EventHandler(Content_AlgorithmNameChanged), sender, e);
162      else UpdateCaption();
163    }
164    private void Content_UpdateOfRunsInProgressChanged(object sender, EventArgs e) {
165      if (InvokeRequired) {
166        Invoke(new EventHandler(Content_UpdateOfRunsInProgressChanged), sender, e);
167        return;
168      }
169      suppressUpdates = Content.UpdateOfRunsInProgress;
170      if (!suppressUpdates) {
171        UpdateDataTableComboBox();
172        UpdateGroupAndProblemComboBox();
173        UpdateRuns();
174      }
175    }
176
177    private void RegisterRunEvents(IRun run) {
178      run.PropertyChanged += run_PropertyChanged;
179    }
180    private void DeregisterRunEvents(IRun run) {
181      run.PropertyChanged -= run_PropertyChanged;
182    }
183    private void run_PropertyChanged(object sender, PropertyChangedEventArgs e) {
184      if (suppressUpdates) return;
185      if (InvokeRequired) {
186        Invoke((Action<object, PropertyChangedEventArgs>)run_PropertyChanged, sender, e);
187      } else {
188        if (e.PropertyName == "Visible")
189          UpdateRuns();
190      }
191    }
192    #endregion
193
194    protected override void OnContentChanged() {
195      base.OnContentChanged();
196      dataTableComboBox.Items.Clear();
197      groupComboBox.Items.Clear();
198      byTargetDataTable.Rows.Clear();
199
200      UpdateCaption();
201      if (Content != null) {
202        UpdateGroupAndProblemComboBox();
203        UpdateDataTableComboBox();
204      }
205    }
206
207
208    private void UpdateGroupAndProblemComboBox() {
209      var selectedGroupItem = (string)groupComboBox.SelectedItem;
210
211      var groupings = Content.ParameterNames.OrderBy(x => x).ToArray();
212      groupComboBox.Items.Clear();
213      groupComboBox.Items.Add(AllRuns);
214      groupComboBox.Items.AddRange(groupings);
215      if (selectedGroupItem != null && groupComboBox.Items.Contains(selectedGroupItem)) {
216        groupComboBox.SelectedItem = selectedGroupItem;
217      } else if (groupComboBox.Items.Count > 0) {
218        groupComboBox.SelectedItem = groupComboBox.Items[0];
219      }
220
221      var problems = new HashSet<ProblemDescription>();
222      foreach (var run in Content) {
223        problems.Add(new ProblemDescription(run));
224      }
225
226      var problemTypesDifferent = problems.Select(x => x.ProblemType).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
227      var problemNamesDifferent = problems.Select(x => x.ProblemName).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
228      var evaluatorDifferent = problems.Select(x => x.Evaluator).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
229      var maximizationDifferent = problems.Select(x => x.Maximization).Distinct().Count() > 1;
230      var allEqual = !problemTypesDifferent && !problemNamesDifferent && !evaluatorDifferent && !maximizationDifferent;
231
232      var selectedProblemItem = (ProblemDescription)problemComboBox.SelectedItem;
233      problemComboBox.Items.Clear();
234      problemComboBox.Items.Add(ProblemDescription.MatchAll);
235      if (selectedProblemItem == null || selectedProblemItem == ProblemDescription.MatchAll)
236        problemComboBox.SelectedIndex = 0;
237      foreach (var prob in problems.OrderBy(x => x.ToString()).ToList()) {
238        prob.DisplayProblemType = problemTypesDifferent;
239        prob.DisplayProblemName = problemNamesDifferent || allEqual;
240        prob.DisplayEvaluator = evaluatorDifferent;
241        prob.DisplayMaximization = maximizationDifferent;
242        problemComboBox.Items.Add(prob);
243        if (prob.Equals(selectedProblemItem)) problemComboBox.SelectedItem = prob;
244      }
245      SetEnabledStateOfControls();
246    }
247
248    private void UpdateDataTableComboBox() {
249      string selectedItem = (string)dataTableComboBox.SelectedItem;
250
251      dataTableComboBox.Items.Clear();
252      var dataTables = (from run in Content
253                        from result in run.Results
254                        where result.Value is IndexedDataTable<double>
255                        select result.Key).Distinct().ToArray();
256
257      dataTableComboBox.Items.AddRange(dataTables);
258      if (selectedItem != null && dataTableComboBox.Items.Contains(selectedItem)) {
259        dataTableComboBox.SelectedItem = selectedItem;
260      } else if (dataTableComboBox.Items.Count > 0) {
261        dataTableComboBox.SelectedItem = dataTableComboBox.Items[0];
262      }
263    }
264
265    protected override void SetEnabledStateOfControls() {
266      base.SetEnabledStateOfControls();
267      groupComboBox.Enabled = Content != null;
268      problemComboBox.Enabled = Content != null && problemComboBox.Items.Count > 1;
269      dataTableComboBox.Enabled = Content != null && dataTableComboBox.Items.Count > 1;
270      addTargetsAsResultButton.Enabled = Content != null && targets != null && dataTableComboBox.SelectedIndex >= 0;
271      addBudgetsAsResultButton.Enabled = Content != null && budgets != null && dataTableComboBox.SelectedIndex >= 0;
272    }
273
274    private Dictionary<string, Dictionary<ProblemDescription, Tuple<double, List<IRun>>>> GroupRuns() {
275      var groupedRuns = new Dictionary<string, Dictionary<ProblemDescription, Tuple<double, List<IRun>>>>();
276
277      var table = (string)dataTableComboBox.SelectedItem;
278      if (string.IsNullOrEmpty(table)) return groupedRuns;
279
280      var selectedGroup = (string)groupComboBox.SelectedItem;
281      if (string.IsNullOrEmpty(selectedGroup)) return groupedRuns;
282
283      var selectedProblem = (ProblemDescription)problemComboBox.SelectedItem;
284      if (selectedProblem == null) return groupedRuns;
285
286      var targetsPerProblem = CalculateBestTargetPerProblemInstance(table);
287
288      foreach (var x in (from r in Content
289                         where (selectedGroup == AllRuns || r.Parameters.ContainsKey(selectedGroup))
290                           && selectedProblem.Match(r)
291                           && r.Results.ContainsKey(table)
292                           && r.Visible
293                         group r by selectedGroup == AllRuns ? AllRuns : r.Parameters[selectedGroup].ToString() into g
294                         select Tuple.Create(g.Key, g.ToList()))) {
295        var pDict = new Dictionary<ProblemDescription, Tuple<double, List<IRun>>>();
296        foreach (var y in (from r in x.Item2
297                           let pd = new ProblemDescription(r)
298                           group r by pd into g
299                           select Tuple.Create(g.Key, g.ToList()))) {
300          pDict[y.Item1] = Tuple.Create(targetsPerProblem[y.Item1], y.Item2);
301        }
302        groupedRuns[x.Item1] = pDict;
303      }
304
305      return groupedRuns;
306    }
307
308    #region Performance analysis by (multiple) target(s)
309    private void UpdateResultsByTarget() {
310      // necessary to reset log scale -> empty chart cannot use log scaling
311      byTargetDataTable.VisualProperties.XAxisLogScale = false;
312      byTargetDataTable.Rows.Clear();
313
314      var table = (string)dataTableComboBox.SelectedItem;
315      if (string.IsNullOrEmpty(table)) return;
316
317      if (targets == null) GenerateDefaultTargets();
318
319      var groupedRuns = GroupRuns();
320      if (groupedRuns.Count == 0) return;
321
322      var xAxisTitles = new HashSet<string>();
323      var colorCount = 0;
324      var lineStyleCount = 0;
325
326      var maxEfforts = new Dictionary<ProblemDescription, double>();
327      foreach (var group in groupedRuns) {
328        foreach (var problem in group.Value) {
329          double eff;
330          if (!maxEfforts.TryGetValue(problem.Key, out eff)) {
331            eff = 0.0;
332          }
333          foreach (var run in problem.Value.Item2) {
334            var maxEff = ((IndexedDataTable<double>)run.Results[table]).Rows.First().Values.Last().Item1;
335            if (maxEff > eff) eff = maxEff;
336          }
337          maxEfforts[problem.Key] = eff;
338        }
339      }
340      // if the group contains multiple different problems we want to use the minimal maximal observed effort
341      // otherwise we run into situations where we don't have data for a certain problem instance anymore
342      // this is a special case when aggregating over multiple problem instances
343      var maxEffort = maxEfforts.Values.Min();
344
345      foreach (var group in groupedRuns) {
346        // hits describes the number of target hits at a certain time for a certain group
347        var hits = new Dictionary<string, SortedList<double, int>>();
348        // misses describes the number of target misses after a certain time for a certain group
349        // for instance when a run ends, but has not achieved all targets, misses describes
350        // how many targets have been left open at the point when the run ended
351        var misses = new Dictionary<string, SortedList<double, int>>();
352        var maxLength = 0.0;
353
354        var noRuns = 0;
355        foreach (var problem in group.Value) {
356          foreach (var run in problem.Value.Item2) {
357            var resultsTable = (IndexedDataTable<double>)run.Results[table];
358            xAxisTitles.Add(resultsTable.VisualProperties.XAxisTitle);
359
360            if (eachOrAllTargetCheckBox.Checked) {
361              CalculateHitsForEachTarget(hits, misses, resultsTable.Rows.First(), problem.Key, group.Key, problem.Value.Item1, maxEffort);
362              // achiving each target can be seen as a separate run for that target only
363              noRuns += targets.Length;
364            } else {
365              var length = CalculateHitsForAllTargets(hits, misses, resultsTable.Rows.First(), problem.Key, group.Key, problem.Value.Item1, maxEffort);
366              maxLength = Math.Max(length, maxLength);
367              noRuns++;
368            }
369          }
370        }
371
372        foreach (var list in hits) {
373          var row = new IndexedDataRow<double>(list.Key) {
374            VisualProperties = {
375              ChartType = DataRowVisualProperties.DataRowChartType.StepLine,
376              LineWidth = 2,
377              Color = colors[colorCount],
378              LineStyle = lineStyles[lineStyleCount],
379              StartIndexZero = false
380            }
381          };
382
383          var ecdf = 0.0;
384          var iter = misses[list.Key].GetEnumerator();
385          iter.MoveNext();
386          var sumTargets = noRuns * targets.Length;
387          foreach (var h in list.Value) {
388            ecdf += h.Value;
389            while (iter.Current.Key < h.Key) {
390              sumTargets -= iter.Current.Value;
391              if (!iter.MoveNext()) break;
392            }
393            row.Values.Add(Tuple.Create(h.Key, ecdf / sumTargets));
394          }
395
396          if (maxLength > 0 && (row.Values.Count == 0 || row.Values.Last().Item1 < maxLength))
397            row.Values.Add(Tuple.Create(maxLength, ecdf / sumTargets));
398
399          byTargetDataTable.Rows.Add(row);
400        }
401        colorCount = (colorCount + 1) % colors.Length;
402        if (colorCount == 0) lineStyleCount = (lineStyleCount + 1) % lineStyles.Length;
403      }
404
405      if (targets.Length == 1)
406        ByTargetDataTable.VisualProperties.YAxisTitle = "Probability to be " + (targets[0] * 100) + "% worse than best";
407      else ByTargetDataTable.VisualProperties.YAxisTitle = "Proportion of reached targets";
408      byTargetDataTable.VisualProperties.XAxisTitle = string.Join(" / ", xAxisTitles);
409      byTargetDataTable.VisualProperties.XAxisLogScale = byTargetDataTable.Rows.Count > 0 && targetLogScalingCheckBox.Checked;
410
411      UpdateErtTables(groupedRuns);
412    }
413
414    private void GenerateDefaultTargets() {
415      targets = new[] { 0.1, 0.05, 0.02, 0.01, 0 };
416      suppressTargetsEvents = true;
417      targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100)) + "%";
418      suppressTargetsEvents = false;
419    }
420
421    private void CalculateHitsForEachTarget(Dictionary<string, SortedList<double, int>> hits,
422                                            Dictionary<string, SortedList<double, int>> misses,
423                                            IndexedDataRow<double> row, ProblemDescription problem,
424                                            string group, double bestTarget, double maxEffort) {
425      foreach (var l in targets.Select(x => (problem.IsMaximization() ? (1 - x) : (1 + x)) * bestTarget)) {
426        var key = group + "-" + l;
427        if (!hits.ContainsKey(key)) {
428          hits.Add(key, new SortedList<double, int>());
429          misses.Add(key, new SortedList<double, int>());
430        }
431        var hit = false;
432        foreach (var v in row.Values) {
433          if (v.Item1 > maxEffort) break;
434          if (problem.IsMaximization() && v.Item2 >= l || !problem.IsMaximization() && v.Item2 <= l) {
435            if (hits[key].ContainsKey(v.Item1))
436              hits[key][v.Item1]++;
437            else hits[key][v.Item1] = 1;
438            hit = true;
439            break;
440          }
441        }
442        if (!hit) {
443          var max = Math.Min(row.Values.Last().Item1, maxEffort);
444          if (misses[key].ContainsKey(max))
445            misses[key][max]++;
446          else misses[key][max] = 1;
447        }
448      }
449    }
450
451    private double CalculateHitsForAllTargets(Dictionary<string, SortedList<double, int>> hits,
452                                              Dictionary<string, SortedList<double, int>> misses,
453                                              IndexedDataRow<double> row, ProblemDescription problem,
454                                              string group, double bestTarget, double maxEffort) {
455      var values = row.Values;
456      if (!hits.ContainsKey(group)) {
457        hits.Add(group, new SortedList<double, int>());
458        misses.Add(group, new SortedList<double, int>());
459      }
460
461      var i = 0;
462      var j = 0;
463      while (i < targets.Length && j < values.Count) {
464        var target = (problem.IsMaximization() ? (1 - targets[i]) : (1 + targets[i])) * bestTarget;
465        var current = values[j];
466        if (current.Item1 > maxEffort) break;
467        if (problem.IsMaximization() && current.Item2 >= target
468          || !problem.IsMaximization() && current.Item2 <= target) {
469          if (hits[group].ContainsKey(current.Item1)) hits[group][current.Item1]++;
470          else hits[group][current.Item1] = 1;
471          i++;
472        } else {
473          j++;
474        }
475      }
476      if (j == values.Count) j--;
477      var effort = Math.Min(values[j].Item1, maxEffort);
478      if (i < targets.Length) {
479        if (misses[group].ContainsKey(effort))
480          misses[group][effort] += targets.Length - i;
481        else misses[group][effort] = targets.Length - i;
482      }
483      return effort;
484    }
485
486    private void UpdateErtTables(Dictionary<string, Dictionary<ProblemDescription, Tuple<double, List<IRun>>>> groupedRuns) {
487      ertTableView.Content = null;
488      var columns = 1 + targets.Length + 1;
489      var matrix = new string[groupedRuns.Count * groupedRuns.Max(x => x.Value.Count) + groupedRuns.Max(x => x.Value.Count), columns];
490      var rowCount = 0;
491
492      var tableName = (string)dataTableComboBox.SelectedItem;
493      if (string.IsNullOrEmpty(tableName)) return;
494
495      var targetsPerProblem = CalculateBestTargetPerProblemInstance(tableName);
496
497      var colNames = new string[columns];
498      colNames[0] = colNames[colNames.Length - 1] = string.Empty;
499      for (var i = 0; i < targets.Length; i++) {
500        colNames[i + 1] = targets[i].ToString("0.0%");
501      }
502      var problems = groupedRuns.SelectMany(x => x.Value.Keys).Distinct().ToList();
503
504      foreach (var problem in problems) {
505        matrix[rowCount, 0] = problem.ToString();
506        for (var i = 0; i < targets.Length; i++) {
507          matrix[rowCount, i + 1] = (targetsPerProblem[problem] * (problem.IsMaximization() ? (1 - targets[i]) : (1 + targets[i]))).ToString(CultureInfo.CurrentCulture.NumberFormat);
508        }
509        matrix[rowCount, columns - 1] = "#succ";
510        rowCount++;
511
512        foreach (var group in groupedRuns) {
513          matrix[rowCount, 0] = group.Key;
514          if (!group.Value.ContainsKey(problem)) {
515            matrix[rowCount, columns - 1] = "N/A";
516            rowCount++;
517            continue;
518          }
519          var runs = group.Value[problem].Item2;
520          ErtCalculationResult result = default(ErtCalculationResult);
521          for (var i = 0; i < targets.Length; i++) {
522            result = ExpectedRuntimeHelper.CalculateErt(runs, tableName, (problem.IsMaximization() ? (1 - targets[i]) : (1 + targets[i])) * group.Value[problem].Item1, problem.IsMaximization());
523            matrix[rowCount, i + 1] = result.ToString();
524          }
525          matrix[rowCount, columns - 1] = targets.Length > 0 ? result.SuccessfulRuns + "/" + result.TotalRuns : "-";
526          rowCount++;
527        }
528      }
529      ertTableView.Content = new StringMatrix(matrix) { ColumnNames = colNames };
530      ertTableView.DataGridView.AutoResizeColumns(DataGridViewAutoSizeColumnsMode.AllCells);
531    }
532    #endregion
533
534    #region Performance analysis by (multiple) budget(s)
535    private void UpdateResultsByCost() {
536      // necessary to reset log scale -> empty chart cannot use log scaling
537      byCostDataTable.VisualProperties.XAxisLogScale = false;
538      byCostDataTable.Rows.Clear();
539
540      var table = (string)dataTableComboBox.SelectedItem;
541      if (string.IsNullOrEmpty(table)) return;
542
543      if (budgets == null) GenerateDefaultBudgets(table);
544
545      var groupedRuns = GroupRuns();
546      if (groupedRuns.Count == 0) return;
547
548      var colorCount = 0;
549      var lineStyleCount = 0;
550
551      var targetsPerProblem = CalculateBestTargetPerProblemInstance((string)dataTableComboBox.SelectedItem);
552
553      foreach (var group in groupedRuns) {
554        var hits = new Dictionary<string, SortedList<double, double>>();
555
556        foreach (var problem in group.Value) {
557          foreach (var run in problem.Value.Item2) {
558            var resultsTable = (IndexedDataTable<double>)run.Results[table];
559
560            if (eachOrAllBudgetsCheckBox.Checked) {
561              CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), group.Value.Count, problem.Key, group.Key, problem.Value.Item2.Count, targetsPerProblem[problem.Key]);
562            } else {
563              CalculateHitsForAllBudgets(hits, resultsTable.Rows.First(), group.Value.Count, problem.Key, group.Key, problem.Value.Item2.Count, targetsPerProblem[problem.Key]);
564            }
565          }
566        }
567
568        foreach (var list in hits) {
569          var row = new IndexedDataRow<double>(list.Key) {
570            VisualProperties = {
571              ChartType = DataRowVisualProperties.DataRowChartType.StepLine,
572              LineWidth = 2,
573              Color = colors[colorCount],
574              LineStyle = lineStyles[lineStyleCount],
575              StartIndexZero = false
576            }
577          };
578
579          var total = 0.0;
580          foreach (var h in list.Value) {
581            total += h.Value;
582            row.Values.Add(Tuple.Create(h.Key, total));
583          }
584
585          byCostDataTable.Rows.Add(row);
586        }
587        colorCount = (colorCount + 1) % colors.Length;
588        if (colorCount == 0) lineStyleCount = (lineStyleCount + 1) % lineStyles.Length;
589      }
590
591      byCostDataTable.VisualProperties.XAxisTitle = "Targets to Best-Known Ratio";
592      byCostDataTable.VisualProperties.XAxisLogScale = byCostDataTable.Rows.Count > 0 && budgetLogScalingCheckBox.Checked;
593    }
594
595    private void GenerateDefaultBudgets(string table) {
596      var runs = GroupRuns().SelectMany(x => x.Value.Values).SelectMany(x => x.Item2).ToList();
597      var min = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Min()).Min();
598      var max = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Max()).Max();
599
600      var maxMagnitude = (int)Math.Ceiling(Math.Log10(max));
601      var minMagnitude = (int)Math.Floor(Math.Log10(min));
602      if (maxMagnitude - minMagnitude >= 3) {
603        budgets = new double[maxMagnitude - minMagnitude];
604        for (var i = minMagnitude; i < maxMagnitude; i++) {
605          budgets[i - minMagnitude] = Math.Pow(10, i);
606        }
607      } else {
608        var range = max - min;
609        budgets = Enumerable.Range(0, 6).Select(x => min + (x / 5.0) * range).ToArray();
610      }
611      suppressBudgetsEvents = true;
612      budgetsTextBox.Text = string.Join(" ; ", budgets);
613      suppressBudgetsEvents = false;
614    }
615
616    private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, ProblemDescription problem, string groupName, int problemCount, double bestTarget) {
617      foreach (var b in budgets) {
618        var key = groupName + "-" + b;
619        if (!hits.ContainsKey(key)) hits.Add(key, new SortedList<double, double>());
620        Tuple<double, double> prev = null;
621        foreach (var v in row.Values) {
622          if (v.Item1 >= b) {
623            // the budget may be too low to achieve any target
624            if (prev == null && v.Item1 != b) break;
625            var tgt = ((prev == null || v.Item1 == b) ? v.Item2 : prev.Item2);
626            tgt = problem.IsMaximization() ? bestTarget / tgt : tgt / bestTarget;
627            if (hits[key].ContainsKey(tgt))
628              hits[key][tgt] += 1.0 / (groupCount * problemCount);
629            else hits[key][tgt] = 1.0 / (groupCount * problemCount);
630            break;
631          }
632          prev = v;
633        }
634        if (hits[key].Count == 0) hits.Remove(key);
635      }
636    }
637
638    private void CalculateHitsForAllBudgets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, ProblemDescription problem, string groupName, int problemCount, double bestTarget) {
639      var values = row.Values;
640      if (!hits.ContainsKey(groupName)) hits.Add(groupName, new SortedList<double, double>());
641
642      var i = 0;
643      var j = 0;
644      Tuple<double, double> prev = null;
645      while (i < budgets.Length && j < values.Count) {
646        var current = values[j];
647        if (current.Item1 >= budgets[i]) {
648          if (prev != null || current.Item1 == budgets[i]) {
649            var tgt = (prev == null || current.Item1 == budgets[i]) ? current.Item2 : prev.Item2;
650            tgt = problem.IsMaximization() ? bestTarget / tgt : tgt / bestTarget;
651            if (!hits[groupName].ContainsKey(tgt)) hits[groupName][tgt] = 0;
652            hits[groupName][tgt] += 1.0 / (groupCount * problemCount * budgets.Length);
653          }
654          i++;
655        } else {
656          j++;
657          prev = current;
658        }
659      }
660      var lastTgt = values.Last().Item2;
661      lastTgt = problem.IsMaximization() ? bestTarget / lastTgt : lastTgt / bestTarget;
662      if (i < budgets.Length && !hits[groupName].ContainsKey(lastTgt)) hits[groupName][lastTgt] = 0;
663      while (i < budgets.Length) {
664        hits[groupName][lastTgt] += 1.0 / (groupCount * problemCount * budgets.Length);
665        i++;
666      }
667    }
668    #endregion
669
670    private void UpdateCaption() {
671      Caption = Content != null ? Content.OptimizerName + " RLD View" : ViewAttribute.GetViewName(GetType());
672    }
673
674    private void groupComboBox_SelectedIndexChanged(object sender, EventArgs e) {
675      UpdateRuns();
676      SetEnabledStateOfControls();
677    }
678    private void problemComboBox_SelectedIndexChanged(object sender, EventArgs e) {
679      UpdateRuns();
680      SetEnabledStateOfControls();
681    }
682    private void dataTableComboBox_SelectedIndexChanged(object sender, EventArgs e) {
683      if (dataTableComboBox.SelectedIndex >= 0)
684        GenerateDefaultBudgets((string)dataTableComboBox.SelectedItem);
685      UpdateRuns();
686      SetEnabledStateOfControls();
687    }
688
689    private void logScalingCheckBox_CheckedChanged(object sender, EventArgs e) {
690      byTargetDataTable.VisualProperties.XAxisLogScale = byTargetDataTable.Rows.Count > 0 && targetLogScalingCheckBox.Checked;
691      byCostDataTable.VisualProperties.XAxisLogScale = byCostDataTable.Rows.Count > 0 && budgetLogScalingCheckBox.Checked;
692    }
693
694    #region Event handlers for target analysis
695    private bool suppressTargetsEvents;
696    private void targetsTextBox_Validating(object sender, CancelEventArgs e) {
697      if (suppressTargetsEvents) return;
698      var targetStrings = targetsTextBox.Text.Split(new[] { '%', ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
699      var targetList = new List<decimal>();
700      foreach (var ts in targetStrings) {
701        decimal t;
702        if (!decimal.TryParse(ts, out t)) {
703          errorProvider.SetError(targetsTextBox, "Not all targets can be parsed: " + ts);
704          e.Cancel = true;
705          return;
706        }
707        targetList.Add(t / 100);
708      }
709      if (targetList.Count == 0) {
710        errorProvider.SetError(targetsTextBox, "Give at least one target value!");
711        e.Cancel = true;
712        return;
713      }
714      e.Cancel = false;
715      errorProvider.SetError(targetsTextBox, null);
716      targets = targetList.Select(x => (double)x).OrderByDescending(x => x).ToArray();
717      suppressTargetsEvents = true;
718      try { targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100)) + "%"; } finally { suppressTargetsEvents = false; }
719
720      UpdateResultsByTarget();
721      SetEnabledStateOfControls();
722    }
723
724    private void eachOrAllTargetCheckBox_CheckedChanged(object sender, EventArgs e) {
725      var each = eachOrAllTargetCheckBox.Checked;
726      eachOrAllTargetCheckBox.Text = each ? "each" : "all";
727      SuspendRepaint();
728      try {
729        UpdateResultsByTarget();
730      } finally { ResumeRepaint(true); }
731    }
732
733    private void generateTargetsButton_Click(object sender, EventArgs e) {
734      decimal max = 1, min = 0, count = 10;
735      if (targets != null) {
736        max = (decimal)targets.Max();
737        min = (decimal)targets.Min();
738        count = targets.Length;
739      } else if (Content.Count > 0 && dataTableComboBox.SelectedIndex >= 0) {
740        var table = (string)dataTableComboBox.SelectedItem;
741        max = (decimal)Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Max(y => y.Item2)).Max();
742        min = (decimal)Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Min(y => y.Item2)).Min();
743        count = 6;
744      }
745      using (var dialog = new DefineArithmeticProgressionDialog(false, min, max, (max - min) / count)) {
746        if (dialog.ShowDialog() == DialogResult.OK) {
747          if (dialog.Values.Any()) {
748            targets = dialog.Values.Select(x => (double)x).ToArray();
749            suppressTargetsEvents = true;
750            targetsTextBox.Text = string.Join("% ; ", targets);
751            suppressTargetsEvents = false;
752
753            UpdateResultsByTarget();
754            SetEnabledStateOfControls();
755          }
756        }
757      }
758    }
759
760    private void addTargetsAsResultButton_Click(object sender, EventArgs e) {
761      var table = (string)dataTableComboBox.SelectedItem;
762
763      var targetsPerProblem = CalculateBestTargetPerProblemInstance(table);
764
765      foreach (var run in Content) {
766        if (!run.Results.ContainsKey(table)) continue;
767        var resultsTable = (IndexedDataTable<double>)run.Results[table];
768        var values = resultsTable.Rows.First().Values;
769        var i = 0;
770        var j = 0;
771        var pd = new ProblemDescription(run);
772        while (i < targets.Length && j < values.Count) {
773          var target = (pd.IsMaximization() ? (1 - targets[i]) : (1 + targets[i])) * targetsPerProblem[pd];
774          var current = values[j];
775          if (pd.IsMaximization() && current.Item2 >= target
776              || !pd.IsMaximization() && current.Item2 <= target) {
777            run.Results[table + ".Target" + target] = new DoubleValue(current.Item1);
778            i++;
779          } else {
780            j++;
781          }
782        }
783      }
784    }
785    #endregion
786
787    #region Event handlers for cost analysis
788    private bool suppressBudgetsEvents;
789    private void budgetsTextBox_Validating(object sender, CancelEventArgs e) {
790      if (suppressBudgetsEvents) return;
791      var budgetStrings = budgetsTextBox.Text.Split(new[] { ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
792      var budgetList = new List<double>();
793      foreach (var ts in budgetStrings) {
794        double b;
795        if (!double.TryParse(ts, out b)) {
796          errorProvider.SetError(budgetsTextBox, "Not all targets can be parsed: " + ts);
797          e.Cancel = true;
798          return;
799        }
800        budgetList.Add(b);
801      }
802      if (budgetList.Count == 0) {
803        errorProvider.SetError(budgetsTextBox, "Give at least one target value!");
804        e.Cancel = true;
805        return;
806      }
807      e.Cancel = false;
808      errorProvider.SetError(budgetsTextBox, null);
809      budgets = budgetList.ToArray();
810      UpdateResultsByCost();
811      SetEnabledStateOfControls();
812    }
813
814    private void eachOrAllBudgetsCheckBox_CheckedChanged(object sender, EventArgs e) {
815      var each = eachOrAllBudgetsCheckBox.Checked;
816      eachOrAllBudgetsCheckBox.Text = each ? "each" : "all";
817      SuspendRepaint();
818      try {
819        UpdateResultsByCost();
820      } finally { ResumeRepaint(true); }
821    }
822
823    private void generateBudgetsButton_Click(object sender, EventArgs e) {
824      decimal max = 1, min = 0, count = 10;
825      if (budgets != null) {
826        max = (decimal)budgets.Max();
827        min = (decimal)budgets.Min();
828        count = budgets.Length;
829      } else if (Content.Count > 0 && dataTableComboBox.SelectedIndex >= 0) {
830        var table = (string)dataTableComboBox.SelectedItem;
831        min = (decimal)Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Min(y => y.Item1)).Min();
832        max = (decimal)Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Max(y => y.Item1)).Max();
833        count = 6;
834      }
835      using (var dialog = new DefineArithmeticProgressionDialog(false, min, max, (max - min) / count)) {
836        if (dialog.ShowDialog() == DialogResult.OK) {
837          if (dialog.Values.Any()) {
838            budgets = dialog.Values.OrderBy(x => x).Select(x => (double)x).ToArray();
839
840            suppressBudgetsEvents = true;
841            budgetsTextBox.Text = string.Join(" ; ", budgets);
842            suppressBudgetsEvents = false;
843
844            UpdateResultsByCost();
845            SetEnabledStateOfControls();
846          }
847        }
848      }
849    }
850
851    private void addBudgetsAsResultButton_Click(object sender, EventArgs e) {
852      var table = (string)dataTableComboBox.SelectedItem;
853      var budgetStrings = budgetsTextBox.Text.Split(new[] { ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
854      if (budgetStrings.Length == 0) {
855        MessageBox.Show("Define a number of budgets.");
856        return;
857      }
858      var budgetList = new List<double>();
859      foreach (var bs in budgetStrings) {
860        double v;
861        if (!double.TryParse(bs, out v)) {
862          MessageBox.Show("Budgets must be a valid number: " + bs);
863          return;
864        }
865        budgetList.Add(v);
866      }
867      budgetList.Sort();
868
869      foreach (var run in Content) {
870        if (!run.Results.ContainsKey(table)) continue;
871        var resultsTable = (IndexedDataTable<double>)run.Results[table];
872        var values = resultsTable.Rows.First().Values;
873        var i = 0;
874        var j = 0;
875        Tuple<double, double> prev = null;
876        while (i < budgetList.Count && j < values.Count) {
877          var current = values[j];
878          if (current.Item1 >= budgetList[i]) {
879            if (prev != null || current.Item1 == budgetList[i]) {
880              var tgt = (prev == null || current.Item1 == budgetList[i]) ? current.Item2 : prev.Item2;
881              run.Results[table + ".Cost" + budgetList[i]] = new DoubleValue(tgt);
882            }
883            i++;
884          } else {
885            j++;
886            prev = current;
887          }
888        }
889      }
890    }
891    #endregion
892
893    #region Helpers
894    private Dictionary<ProblemDescription, double> CalculateBestTargetPerProblemInstance(string table) {
895      return (from r in Content
896              where r.Visible
897              let pd = new ProblemDescription(r)
898              let target = r.Parameters.ContainsKey("BestKnownQuality")
899                           && r.Parameters["BestKnownQuality"] is DoubleValue
900                ? ((DoubleValue)r.Parameters["BestKnownQuality"]).Value
901                : ((IndexedDataTable<double>)r.Results[table]).Rows.First().Values.Last().Item2
902              group target by pd into g
903              select new { Problem = g.Key, Target = g.Key.IsMaximization() ? g.Max() : g.Min() })
904        .ToDictionary(x => x.Problem, x => x.Target);
905    }
906
907    private void UpdateRuns() {
908      if (InvokeRequired) {
909        Invoke((Action)UpdateRuns);
910        return;
911      }
912      SuspendRepaint();
913      try {
914        UpdateResultsByTarget();
915        UpdateResultsByCost();
916      } finally { ResumeRepaint(true); }
917    }
918    #endregion
919
920
921    private class ProblemDescription {
922      private readonly bool matchAll;
923      public static readonly ProblemDescription MatchAll = new ProblemDescription() {
924        ProblemName = "All with Best-Known"
925      };
926
927      private ProblemDescription() {
928        ProblemType = string.Empty;
929        ProblemName = string.Empty;
930        Evaluator = string.Empty;
931        Maximization = string.Empty;
932        DisplayProblemType = false;
933        DisplayProblemName = false;
934        DisplayEvaluator = false;
935        DisplayMaximization = false;
936        matchAll = true;
937      }
938
939      public ProblemDescription(IRun run) {
940        ProblemType = GetStringValueOrEmpty(run, "Problem Type");
941        ProblemName = GetStringValueOrEmpty(run, "Problem Name");
942        Evaluator = GetStringValueOrEmpty(run, "Evaluator");
943        Maximization = GetMaximizationValueOrEmpty(run, "Maximization");
944        DisplayProblemType = !string.IsNullOrEmpty(ProblemType);
945        DisplayProblemName = !string.IsNullOrEmpty(ProblemName);
946        DisplayEvaluator = !string.IsNullOrEmpty(Evaluator);
947        DisplayMaximization = !string.IsNullOrEmpty(Maximization);
948        matchAll = false;
949      }
950
951      public bool DisplayProblemType { get; set; }
952      public string ProblemType { get; set; }
953      public bool DisplayProblemName { get; set; }
954      public string ProblemName { get; set; }
955      public bool DisplayEvaluator { get; set; }
956      public string Evaluator { get; set; }
957      public bool DisplayMaximization { get; set; }
958      public string Maximization { get; set; }
959
960      public bool IsMaximization() {
961        return Maximization == "MAX";
962      }
963
964      public bool Match(IRun run) {
965        return matchAll ||
966               GetStringValueOrEmpty(run, "Problem Type") == ProblemType
967               && GetStringValueOrEmpty(run, "Problem Name") == ProblemName
968               && GetStringValueOrEmpty(run, "Evaluator") == Evaluator
969               && GetMaximizationValueOrEmpty(run, "Maximization") == Maximization;
970      }
971
972      private string GetStringValueOrEmpty(IRun run, string key) {
973        return run.Parameters.ContainsKey(key) ? ((StringValue)run.Parameters[key]).Value : string.Empty;
974      }
975
976      private string GetMaximizationValueOrEmpty(IRun run, string key) {
977        return run.Parameters.ContainsKey(key) ? (((BoolValue)run.Parameters[key]).Value ? "MAX" : "MIN") : string.Empty;
978      }
979
980      public override bool Equals(object obj) {
981        var other = obj as ProblemDescription;
982        if (other == null) return false;
983        return ProblemType == other.ProblemType
984               && ProblemName == other.ProblemName
985               && Evaluator == other.Evaluator
986               && Maximization == other.Maximization;
987      }
988
989      public override int GetHashCode() {
990        return ProblemType.GetHashCode() ^ ProblemName.GetHashCode() ^ Evaluator.GetHashCode() ^ Maximization.GetHashCode();
991      }
992
993      public override string ToString() {
994        return string.Join("  --  ", new[] {
995          (DisplayProblemType ? ProblemType : string.Empty),
996          (DisplayProblemName ? ProblemName : string.Empty),
997          (DisplayEvaluator ? Evaluator : string.Empty),
998          (DisplayMaximization ? Maximization : string.Empty)}.Where(x => !string.IsNullOrEmpty(x)));
999      }
1000    }
1001  }
1002}
Note: See TracBrowser for help on using the repository browser.