Free cookie consent management tool by TermsFeed Policy Generator

source: branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs @ 12864

Last change on this file since 12864 was 12864, checked in by abeham, 9 years ago

#2431:

  • fixed bug in iterated algorithm
  • changed RLD view to use % deviation as targets
  • RLD view now always groups by problem
File size: 40.7 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.ComponentModel;
25using System.Drawing;
26using System.Globalization;
27using System.Linq;
28using System.Windows.Forms;
29using HeuristicLab.Analysis;
30using HeuristicLab.Collections;
31using HeuristicLab.Core.Views;
32using HeuristicLab.Data;
33using HeuristicLab.MainForm;
34using HeuristicLab.MainForm.WindowsForms;
35
36namespace HeuristicLab.Optimization.Views {
37  [View("Run-length Distribution View")]
38  [Content(typeof(RunCollection), false)]
39  public partial class RunCollectionRLDView : ItemView {
40    private const string AllRuns = "All Runs";
41
42    private static readonly Color[] colors = new[] {
43      Color.FromArgb(0x40, 0x6A, 0xB7),
44      Color.FromArgb(0xB1, 0x6D, 0x01),
45      Color.FromArgb(0x4E, 0x8A, 0x06),
46      Color.FromArgb(0x75, 0x50, 0x7B),
47      Color.FromArgb(0x72, 0x9F, 0xCF),
48      Color.FromArgb(0xA4, 0x00, 0x00),
49      Color.FromArgb(0xAD, 0x7F, 0xA8),
50      Color.FromArgb(0x29, 0x50, 0xCF),
51      Color.FromArgb(0x90, 0xB0, 0x60),
52      Color.FromArgb(0xF5, 0x89, 0x30),
53      Color.FromArgb(0x55, 0x57, 0x53),
54      Color.FromArgb(0xEF, 0x59, 0x59),
55      Color.FromArgb(0xED, 0xD4, 0x30),
56      Color.FromArgb(0x63, 0xC2, 0x16),
57    };
58    private static readonly DataRowVisualProperties.DataRowLineStyle[] lineStyles = new[] {
59      DataRowVisualProperties.DataRowLineStyle.Solid,
60      DataRowVisualProperties.DataRowLineStyle.Dash,
61      DataRowVisualProperties.DataRowLineStyle.DashDot,
62      DataRowVisualProperties.DataRowLineStyle.Dot
63    };
64
65    public new RunCollection Content {
66      get { return (RunCollection)base.Content; }
67      set { base.Content = value; }
68    }
69
70    private double[] targets;
71    private double[] budgets;
72
73    private bool suppressUpdates;
74    private readonly IndexedDataTable<double> byTargetDataTable;
75    public IndexedDataTable<double> ByTargetDataTable {
76      get { return byTargetDataTable; }
77    }
78    private readonly IndexedDataTable<double> byCostDataTable;
79    public IndexedDataTable<double> ByCostDataTable {
80      get { return byCostDataTable; }
81    }
82
83    public RunCollectionRLDView() {
84      InitializeComponent();
85      byTargetDataTable = new IndexedDataTable<double>("ECDF by Target", "A data table containing the ECDF of each of a number of groups.") {
86        VisualProperties = {
87          YAxisTitle = "Proportion of reached targets",
88          YAxisMinimumFixedValue = 0, YAxisMinimumAuto = false,
89          YAxisMaximumFixedValue = 1, YAxisMaximumAuto = false
90        }
91      };
92      byTargetViewHost.Content = byTargetDataTable;
93      byCostDataTable = new IndexedDataTable<double>("ECDF by Cost", "A data table containing the ECDF of each of a number of groups.") {
94        VisualProperties = {
95          YAxisTitle = "Proportion of required budgets",
96          YAxisMinimumFixedValue = 0, YAxisMinimumAuto = false,
97          YAxisMaximumFixedValue = 1, YAxisMaximumAuto = false
98        }
99      };
100      byCostViewHost.Content = byCostDataTable;
101      suppressUpdates = false;
102    }
103
104    #region Content events
105    protected override void RegisterContentEvents() {
106      base.RegisterContentEvents();
107      Content.ItemsAdded += Content_ItemsAdded;
108      Content.ItemsRemoved += Content_ItemsRemoved;
109      Content.CollectionReset += Content_CollectionReset;
110      Content.UpdateOfRunsInProgressChanged += Content_UpdateOfRunsInProgressChanged;
111      Content.OptimizerNameChanged += Content_AlgorithmNameChanged;
112    }
113    protected override void DeregisterContentEvents() {
114      Content.ItemsAdded -= Content_ItemsAdded;
115      Content.ItemsRemoved -= Content_ItemsRemoved;
116      Content.CollectionReset -= Content_CollectionReset;
117      Content.UpdateOfRunsInProgressChanged -= Content_UpdateOfRunsInProgressChanged;
118      Content.OptimizerNameChanged -= Content_AlgorithmNameChanged;
119      base.DeregisterContentEvents();
120    }
121
122    private void Content_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IRun> e) {
123      if (suppressUpdates) return;
124      if (InvokeRequired) {
125        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_ItemsAdded), sender, e);
126        return;
127      }
128      UpdateGroupAndProblemComboBox();
129      UpdateDataTableComboBox();
130      foreach (var run in e.Items)
131        RegisterRunEvents(run);
132    }
133    private void Content_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IRun> e) {
134      if (suppressUpdates) return;
135      if (InvokeRequired) {
136        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_ItemsRemoved), sender, e);
137        return;
138      }
139      UpdateGroupAndProblemComboBox();
140      UpdateDataTableComboBox();
141      foreach (var run in e.Items)
142        DeregisterRunEvents(run);
143    }
144    private void Content_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
145      if (suppressUpdates) return;
146      if (InvokeRequired) {
147        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset), sender, e);
148        return;
149      }
150      UpdateGroupAndProblemComboBox();
151      UpdateDataTableComboBox();
152      foreach (var run in e.OldItems)
153        DeregisterRunEvents(run);
154    }
155    private void Content_AlgorithmNameChanged(object sender, EventArgs e) {
156      if (InvokeRequired)
157        Invoke(new EventHandler(Content_AlgorithmNameChanged), sender, e);
158      else UpdateCaption();
159    }
160    private void Content_UpdateOfRunsInProgressChanged(object sender, EventArgs e) {
161      if (InvokeRequired) {
162        Invoke(new EventHandler(Content_UpdateOfRunsInProgressChanged), sender, e);
163        return;
164      }
165      suppressUpdates = Content.UpdateOfRunsInProgress;
166      if (!suppressUpdates) {
167        UpdateDataTableComboBox();
168        UpdateGroupAndProblemComboBox();
169        UpdateRuns();
170      }
171    }
172
173    private void RegisterRunEvents(IRun run) {
174      run.PropertyChanged += run_PropertyChanged;
175    }
176    private void DeregisterRunEvents(IRun run) {
177      run.PropertyChanged -= run_PropertyChanged;
178    }
179    private void run_PropertyChanged(object sender, PropertyChangedEventArgs e) {
180      if (suppressUpdates) return;
181      if (InvokeRequired) {
182        Invoke((Action<object, PropertyChangedEventArgs>)run_PropertyChanged, sender, e);
183      } else {
184        if (e.PropertyName == "Visible")
185          UpdateRuns();
186      }
187    }
188    #endregion
189
190    protected override void OnContentChanged() {
191      base.OnContentChanged();
192      dataTableComboBox.Items.Clear();
193      groupComboBox.Items.Clear();
194      byTargetDataTable.Rows.Clear();
195
196      UpdateCaption();
197      if (Content != null) {
198        UpdateGroupAndProblemComboBox();
199        UpdateDataTableComboBox();
200      }
201    }
202
203
204    private void UpdateGroupAndProblemComboBox() {
205      var selectedGroupItem = (string)groupComboBox.SelectedItem;
206
207      var groupings = Content.ParameterNames.OrderBy(x => x).ToArray();
208      groupComboBox.Items.Clear();
209      groupComboBox.Items.Add(AllRuns);
210      groupComboBox.Items.AddRange(groupings);
211      if (selectedGroupItem != null && groupComboBox.Items.Contains(selectedGroupItem)) {
212        groupComboBox.SelectedItem = selectedGroupItem;
213      } else if (groupComboBox.Items.Count > 0) {
214        groupComboBox.SelectedItem = groupComboBox.Items[0];
215      }
216
217      var problems = new HashSet<ProblemDescription>();
218      foreach (var run in Content) {
219        problems.Add(new ProblemDescription(run));
220      }
221
222      var problemTypesDifferent = problems.Select(x => x.ProblemType).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
223      var problemNamesDifferent = problems.Select(x => x.ProblemName).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
224      var evaluatorDifferent = problems.Select(x => x.Evaluator).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
225      var allEqual = !problemTypesDifferent && !problemNamesDifferent && !evaluatorDifferent;
226
227      var selectedProblemItem = (ProblemDescription)problemComboBox.SelectedItem;
228      problemComboBox.Items.Clear();
229      problemComboBox.Items.Add(ProblemDescription.MatchAll);
230      if (selectedProblemItem == null || selectedProblemItem == ProblemDescription.MatchAll)
231        problemComboBox.SelectedIndex = 0;
232      foreach (var prob in problems.OrderBy(x => x.ToString()).ToList()) {
233        prob.DisplayProblemType = problemTypesDifferent;
234        prob.DisplayProblemName = problemNamesDifferent || allEqual;
235        prob.DisplayEvaluator = evaluatorDifferent;
236        problemComboBox.Items.Add(prob);
237        if (prob.Equals(selectedProblemItem)) problemComboBox.SelectedItem = prob;
238      }
239      SetEnabledStateOfControls();
240    }
241
242    private void UpdateDataTableComboBox() {
243      string selectedItem = (string)dataTableComboBox.SelectedItem;
244
245      dataTableComboBox.Items.Clear();
246      var dataTables = (from run in Content
247                        from result in run.Results
248                        where result.Value is IndexedDataTable<double>
249                        select result.Key).Distinct().ToArray();
250
251      dataTableComboBox.Items.AddRange(dataTables);
252      if (selectedItem != null && dataTableComboBox.Items.Contains(selectedItem)) {
253        dataTableComboBox.SelectedItem = selectedItem;
254      } else if (dataTableComboBox.Items.Count > 0) {
255        dataTableComboBox.SelectedItem = dataTableComboBox.Items[0];
256      }
257    }
258
259    protected override void SetEnabledStateOfControls() {
260      base.SetEnabledStateOfControls();
261      groupComboBox.Enabled = Content != null;
262      problemComboBox.Enabled = Content != null && problemComboBox.Items.Count > 1;
263      dataTableComboBox.Enabled = Content != null && dataTableComboBox.Items.Count > 1;
264      addTargetsAsResultButton.Enabled = Content != null && targets != null && dataTableComboBox.SelectedIndex >= 0;
265      addBudgetsAsResultButton.Enabled = Content != null && budgets != null && dataTableComboBox.SelectedIndex >= 0;
266    }
267
268    private Dictionary<string, Dictionary<string, Tuple<double, List<IRun>>>> GroupRuns() {
269      var groupedRuns = new Dictionary<string, Dictionary<string, Tuple<double, List<IRun>>>>();
270
271      var table = (string)dataTableComboBox.SelectedItem;
272      if (string.IsNullOrEmpty(table)) return groupedRuns;
273
274      var selectedGroup = (string)groupComboBox.SelectedItem;
275      if (string.IsNullOrEmpty(selectedGroup)) return groupedRuns;
276
277      var selectedProblem = (ProblemDescription)problemComboBox.SelectedItem;
278      if (selectedProblem == null) return groupedRuns;
279
280      var maximization = IsMaximization();
281
282      var targetsPerProblem = (from r in Content
283                               let pd = new ProblemDescription(r).ToString()
284                               let target = r.Parameters.ContainsKey("BestKnownQuality")
285                                && r.Parameters["BestKnownQuality"] is DoubleValue
286                                  ? ((DoubleValue)r.Parameters["BestKnownQuality"]).Value
287                                  : ((IndexedDataTable<double>)r.Results[table]).Rows.First().Values.Last().Item2
288                               group target by pd into g
289                               select new { Problem = g.Key, Target = maximization ? g.Max() : g.Min() })
290                              .ToDictionary(x => x.Problem, x => x.Target);
291
292      foreach (var x in (from r in Content
293                         where selectedGroup == AllRuns || r.Parameters.ContainsKey(selectedGroup)
294                           && selectedProblem.Match(r)
295                           && r.Results.ContainsKey(table)
296                           && r.Visible
297                         group r by selectedGroup == AllRuns ? AllRuns : r.Parameters[selectedGroup].ToString() into g
298                         select Tuple.Create(g.Key, g.ToList()))) {
299        var pDict = new Dictionary<string, Tuple<double, List<IRun>>>();
300        foreach (var y in (from r in x.Item2
301                           let pd = new ProblemDescription(r).ToString()
302                           group r by pd into g
303                           select Tuple.Create(g.Key, g.ToList()))) {
304          pDict[y.Item1] = Tuple.Create(targetsPerProblem[y.Item1], y.Item2);
305        }
306        groupedRuns[x.Item1] = pDict;
307      }
308
309      return groupedRuns;
310    }
311
312    private void UpdateRuns() {
313      if (InvokeRequired) {
314        Invoke((Action)UpdateRuns);
315        return;
316      }
317      SuspendRepaint();
318      try {
319        UpdateResultsByTarget();
320        UpdateResultsByCost();
321      } finally { ResumeRepaint(true); }
322    }
323
324    #region Performance analysis by (multiple) target(s)
325    private void UpdateResultsByTarget() {
326      // necessary to reset log scale -> empty chart cannot use log scaling
327      byTargetDataTable.VisualProperties.XAxisLogScale = false;
328      byTargetDataTable.Rows.Clear();
329
330      var table = (string)dataTableComboBox.SelectedItem;
331      if (string.IsNullOrEmpty(table)) return;
332
333      if (targets == null) GenerateDefaultTargets();
334
335      var groupedRuns = GroupRuns();
336      if (groupedRuns.Count == 0) return;
337
338      var xAxisTitles = new HashSet<string>();
339      var colorCount = 0;
340      var lineStyleCount = 0;
341      var maximization = IsMaximization();
342
343      foreach (var group in groupedRuns) {
344        var hits = new Dictionary<string, SortedList<double, double>>();
345        var maxLength = 0.0;
346
347        foreach (var problem in group.Value) {
348          foreach (var run in problem.Value.Item2) {
349            var resultsTable = (IndexedDataTable<double>)run.Results[table];
350            xAxisTitles.Add(resultsTable.VisualProperties.XAxisTitle);
351
352            if (eachOrAllTargetCheckBox.Checked) {
353              CalculateHitsForEachTarget(hits, resultsTable.Rows.First(), maximization, group.Key, group.Value.Count, problem.Value.Item1, problem.Value.Item2.Count);
354            } else {
355              maxLength = CalculateHitsForAllTargets(hits, resultsTable.Rows.First(), maximization, group.Key, group.Value.Count, problem.Value.Item1, problem.Value.Item2.Count);
356            }
357          }
358        }
359
360        foreach (var list in hits) {
361          var row = new IndexedDataRow<double>(list.Key) {
362            VisualProperties = {
363              ChartType = DataRowVisualProperties.DataRowChartType.StepLine,
364              LineWidth = 2,
365              Color = colors[colorCount],
366              LineStyle = lineStyles[lineStyleCount]
367            }
368          };
369
370          var total = 0.0;
371          foreach (var h in list.Value) {
372            total += h.Value;
373            row.Values.Add(Tuple.Create(h.Key, total));
374          }
375
376          if (maxLength > 0 && (row.Values.Count == 0 || row.Values.Last().Item1 < maxLength))
377            row.Values.Add(Tuple.Create(maxLength, total));
378
379          byTargetDataTable.Rows.Add(row);
380        }
381        colorCount = (colorCount + 1) % colors.Length;
382        if (colorCount == 0) lineStyleCount = (lineStyleCount + 1) % lineStyles.Length;
383      }
384
385      byTargetDataTable.VisualProperties.XAxisTitle = string.Join(" / ", xAxisTitles);
386      byTargetDataTable.VisualProperties.XAxisLogScale = byTargetDataTable.Rows.Count > 0 && targetLogScalingCheckBox.Checked;
387
388      UpdateErtTables(groupedRuns);
389    }
390
391    private void GenerateDefaultTargets() {
392      targets = new[] { 0.1, 0.05, 0.02, 0.01, 0 };
393      suppressTargetsEvents = true;
394      targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100));
395      suppressTargetsEvents = false;
396    }
397
398    private void CalculateHitsForEachTarget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, bool maximization, string group, int groupCount, double bestTarget, int problemCount) {
399      foreach (var l in targets.Select(x => (maximization ? (1 - x) : (1 + x)) * bestTarget)) {
400        var key = group + "-" + l;
401        if (!hits.ContainsKey(key)) hits.Add(key, new SortedList<double, double>());
402        foreach (var v in row.Values) {
403          if (maximization && v.Item2 >= l || !maximization && v.Item2 <= l) {
404            if (hits[key].ContainsKey(v.Item1))
405              hits[key][v.Item1] += 1.0 / (groupCount * problemCount);
406            else hits[key][v.Item1] = 1.0 / (groupCount * problemCount);
407            break;
408          }
409        }
410      }
411    }
412
413    private double CalculateHitsForAllTargets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, bool maximization, string group, int groupCount, double bestTarget, int problemCount) {
414      var values = row.Values;
415      if (!hits.ContainsKey(group)) hits.Add(group, new SortedList<double, double>());
416
417      var i = 0;
418      var j = 0;
419      while (i < targets.Length && j < values.Count) {
420        var target = (maximization ? (1 - targets[i]) : (1 + targets[i])) * bestTarget;
421        var current = values[j];
422        if (maximization && current.Item2 >= target
423          || !maximization && current.Item2 <= target) {
424          if (!hits[group].ContainsKey(current.Item1)) hits[group][current.Item1] = 0;
425          hits[group][current.Item1] += 1.0 / (groupCount * problemCount * targets.Length);
426          i++;
427        } else {
428          j++;
429        }
430      }
431      if (j == values.Count) j--;
432      return values[j].Item1;
433    }
434
435    private void UpdateErtTables(Dictionary<string, Dictionary<string, Tuple<double, List<IRun>>>> groupedRuns) {
436      ertViewHost.Content = null;
437      var columns = 1 + targets.Length + 1;
438      var matrix = new string[groupedRuns.Count * groupedRuns.Max(x => x.Value.Count) + groupedRuns.Max(x => x.Value.Count), columns];
439      var rowCount = 0;
440      var maximization = IsMaximization();
441
442
443      var targetsPerProblem = (from r in Content
444                               let pd = new ProblemDescription(r).ToString()
445                               let target = r.Parameters.ContainsKey("BestKnownQuality")
446                                && r.Parameters["BestKnownQuality"] is DoubleValue
447                                  ? ((DoubleValue)r.Parameters["BestKnownQuality"]).Value
448                                  : ((IndexedDataTable<double>)r.Results[(string)dataTableComboBox.SelectedItem]).Rows.First().Values.Last().Item2
449                               group target by pd into g
450                               select new { Problem = g.Key, Target = maximization ? g.Max() : g.Min() })
451                              .ToDictionary(x => x.Problem, x => x.Target);
452
453      var colNames = new string[columns];
454      colNames[0] = colNames[colNames.Length - 1] = string.Empty;
455      for (var i = 0; i < targets.Length; i++) {
456        colNames[i + 1] = targets[i].ToString("0.0%");
457      }
458      var problems = groupedRuns.SelectMany(x => x.Value.Keys).Distinct().ToList();
459
460      foreach (var problem in problems) {
461        matrix[rowCount, 0] = problem;
462        for (var i = 0; i < targets.Length; i++) {
463          matrix[rowCount, i + 1] = (targetsPerProblem[problem] * (maximization ? (1 - targets[i]) : (1 + targets[i]))).ToString(CultureInfo.CurrentCulture.NumberFormat);
464        }
465        matrix[rowCount, columns - 1] = "#succ";
466        rowCount++;
467
468        foreach (var group in groupedRuns) {
469          matrix[rowCount, 0] = group.Key;
470          var runs = group.Value[problem].Item2;
471          var bestSucc = string.Empty;
472          for (var i = 0; i < targets.Length; i++) {
473            string succ;
474            matrix[rowCount, i + 1] = CalculateExpectedRunTime(runs, (maximization ? (1 - targets[i]) : (1 + targets[i])) * group.Value[problem].Item1, maximization, out succ);
475            if (i == targets.Length - 1) bestSucc = succ;
476          }
477          matrix[rowCount, columns - 1] = bestSucc;
478          rowCount++;
479        }
480      }
481      ertViewHost.Content = new StringMatrix(matrix) { ColumnNames = colNames };
482    }
483
484    private string CalculateExpectedRunTime(List<IRun> group, double target, bool maximization, out string successProb) {
485      var table = (string)dataTableComboBox.SelectedItem;
486      successProb = "-";
487      if (string.IsNullOrEmpty(table)) return "N/A";
488      var successful = new List<double>();
489      var unsuccessful = new List<double>();
490      foreach (var r in group) {
491        var targetAchieved = false;
492        var row = ((IndexedDataTable<double>)r.Results[table]).Rows.First();
493        foreach (var v in row.Values) {
494          if (maximization && v.Item2 >= target || !maximization && v.Item2 <= target) {
495            successful.Add(v.Item1);
496            targetAchieved = true;
497            break;
498          }
499        }
500        if (!targetAchieved) unsuccessful.Add(row.Values.Last().Item1);
501      }
502      successProb = successful.Count + "/" + (successful.Count + unsuccessful.Count);
503      if (successful.Count == 0) return "\u221e"; // infinity symbol
504      if (unsuccessful.Count == 0) return successful.Average().ToString("##,0.0", CultureInfo.CurrentCulture.NumberFormat);
505
506      var ps = successful.Count / (double)(successful.Count + unsuccessful.Count);
507      return (successful.Average() + ((1.0 - ps) / ps) * unsuccessful.Average()).ToString("##,0.0", CultureInfo.CurrentCulture.NumberFormat);
508    }
509    #endregion
510
511    #region Performance analysis by (multiple) budget(s)
512    private void UpdateResultsByCost() {
513      // necessary to reset log scale -> empty chart cannot use log scaling
514      byCostDataTable.VisualProperties.XAxisLogScale = false;
515      byCostDataTable.Rows.Clear();
516
517      var table = (string)dataTableComboBox.SelectedItem;
518      if (string.IsNullOrEmpty(table)) return;
519
520      if (budgets == null) GenerateDefaultBudgets(table);
521
522      var groupedRuns = GroupRuns();
523      if (groupedRuns.Count == 0) return;
524
525      var colorCount = 0;
526      var lineStyleCount = 0;
527
528      foreach (var group in groupedRuns) {
529        var hits = new Dictionary<string, SortedList<double, double>>();
530
531        foreach (var problem in group.Value) {
532          foreach (var run in problem.Value.Item2) {
533            var resultsTable = (IndexedDataTable<double>)run.Results[table];
534
535            if (eachOrAllBudgetsCheckBox.Checked) {
536              CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), group.Value.Count, group.Key, problem.Value.Item2.Count);
537            } else {
538              CalculateHitsForAllBudgets(hits, resultsTable.Rows.First(), group.Value.Count, group.Key, problem.Value.Item2.Count);
539            }
540          }
541        }
542
543        foreach (var list in hits) {
544          var row = new IndexedDataRow<double>(list.Key) {
545            VisualProperties = {
546              ChartType = DataRowVisualProperties.DataRowChartType.StepLine,
547              LineWidth = 2,
548              Color = colors[colorCount],
549              LineStyle = lineStyles[lineStyleCount]
550            }
551          };
552
553          var total = 0.0;
554          foreach (var h in list.Value) {
555            total += h.Value;
556            row.Values.Add(Tuple.Create(h.Key, total));
557          }
558
559          byCostDataTable.Rows.Add(row);
560        }
561        colorCount = (colorCount + 1) % colors.Length;
562        if (colorCount == 0) lineStyleCount = (lineStyleCount + 1) % lineStyles.Length;
563      }
564
565      byCostDataTable.VisualProperties.XAxisTitle = "Targets";
566      byCostDataTable.VisualProperties.XAxisLogScale = byCostDataTable.Rows.Count > 0 && budgetLogScalingCheckBox.Checked;
567    }
568
569    private void GenerateDefaultBudgets(string table) {
570      var runs = GroupRuns().SelectMany(x => x.Value.Values).SelectMany(x => x.Item2).ToList();
571      var min = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Min()).Min();
572      var max = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Max()).Max();
573
574      var maxMagnitude = (int)Math.Ceiling(Math.Log10(max));
575      var minMagnitude = (int)Math.Floor(Math.Log10(min));
576      if (maxMagnitude - minMagnitude >= 3) {
577        budgets = new double[maxMagnitude - minMagnitude];
578        for (var i = minMagnitude; i < maxMagnitude; i++) {
579          budgets[i - minMagnitude] = Math.Pow(10, i);
580        }
581      } else {
582        var range = max - min;
583        budgets = Enumerable.Range(0, 6).Select(x => min + (x / 5.0) * range).ToArray();
584      }
585      suppressBudgetsEvents = true;
586      budgetsTextBox.Text = string.Join(" ; ", budgets);
587      suppressBudgetsEvents = false;
588    }
589
590    private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, string groupName, int problemCount) {
591      foreach (var b in budgets) {
592        var key = groupName + "-" + b;
593        if (!hits.ContainsKey(key)) hits.Add(key, new SortedList<double, double>());
594        Tuple<double, double> prev = null;
595        foreach (var v in row.Values) {
596          if (v.Item1 >= b) {
597            // the budget may be too low to achieve any target
598            if (prev == null && v.Item1 != b) break;
599            var tgt = (prev == null || v.Item1 == b) ? v.Item2 : prev.Item2;
600            if (hits[key].ContainsKey(tgt))
601              hits[key][tgt] += 1.0 / (groupCount * problemCount);
602            else hits[key][tgt] = 1.0 / (groupCount * problemCount);
603            break;
604          }
605          prev = v;
606        }
607        if (hits[key].Count == 0) hits.Remove(key);
608      }
609    }
610
611    private void CalculateHitsForAllBudgets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, string groupName, int problemCount) {
612      var values = row.Values;
613      if (!hits.ContainsKey(groupName)) hits.Add(groupName, new SortedList<double, double>());
614
615      var i = 0;
616      var j = 0;
617      Tuple<double, double> prev = null;
618      while (i < budgets.Length && j < values.Count) {
619        var current = values[j];
620        if (current.Item1 >= budgets[i]) {
621          if (prev != null || current.Item1 == budgets[i]) {
622            var tgt = (prev == null || current.Item1 == budgets[i]) ? current.Item2 : prev.Item2;
623            if (!hits[groupName].ContainsKey(tgt)) hits[groupName][tgt] = 0;
624            hits[groupName][tgt] += 1.0 / (groupCount * problemCount * budgets.Length);
625          }
626          i++;
627        } else {
628          j++;
629          prev = current;
630        }
631      }
632      var lastTgt = values.Last().Item2;
633      if (i < budgets.Length && !hits[groupName].ContainsKey(lastTgt)) hits[groupName][lastTgt] = 0;
634      while (i < budgets.Length) {
635        hits[groupName][lastTgt] += 1.0 / (groupCount * problemCount * budgets.Length);
636        i++;
637      }
638    }
639    #endregion
640
641    private void UpdateCaption() {
642      Caption = Content != null ? Content.OptimizerName + " RLD View" : ViewAttribute.GetViewName(GetType());
643    }
644
645    private void groupComboBox_SelectedIndexChanged(object sender, EventArgs e) {
646      UpdateRuns();
647      SetEnabledStateOfControls();
648    }
649    private void problemComboBox_SelectedIndexChanged(object sender, EventArgs e) {
650      UpdateRuns();
651      SetEnabledStateOfControls();
652    }
653    private void dataTableComboBox_SelectedIndexChanged(object sender, EventArgs e) {
654      if (dataTableComboBox.SelectedIndex >= 0)
655        GenerateDefaultBudgets((string)dataTableComboBox.SelectedItem);
656      UpdateRuns();
657      SetEnabledStateOfControls();
658    }
659
660    private void logScalingCheckBox_CheckedChanged(object sender, EventArgs e) {
661      byTargetDataTable.VisualProperties.XAxisLogScale = byTargetDataTable.Rows.Count > 0 && targetLogScalingCheckBox.Checked;
662      byCostDataTable.VisualProperties.XAxisLogScale = byCostDataTable.Rows.Count > 0 && budgetLogScalingCheckBox.Checked;
663    }
664
665    #region Event handlers for target analysis
666    private bool suppressTargetsEvents;
667    private void targetsTextBox_Validating(object sender, CancelEventArgs e) {
668      if (suppressTargetsEvents) return;
669      var targetStrings = targetsTextBox.Text.Split(new[] { '%', ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
670      var targetList = new List<decimal>();
671      foreach (var ts in targetStrings) {
672        decimal t;
673        if (!decimal.TryParse(ts, out t)) {
674          errorProvider.SetError(targetsTextBox, "Not all targets can be parsed: " + ts);
675          e.Cancel = true;
676          return;
677        }
678        targetList.Add(t / 100);
679      }
680      if (targetList.Count == 0) {
681        errorProvider.SetError(targetsTextBox, "Give at least one target value!");
682        e.Cancel = true;
683        return;
684      }
685      e.Cancel = false;
686      errorProvider.SetError(targetsTextBox, null);
687      targets = targetList.Select(x => (double)x).ToArray();
688      UpdateResultsByTarget();
689      SetEnabledStateOfControls();
690    }
691
692    private void eachOrAllTargetCheckBox_CheckedChanged(object sender, EventArgs e) {
693      var each = eachOrAllTargetCheckBox.Checked;
694      eachOrAllTargetCheckBox.Text = each ? "each" : "all";
695      SuspendRepaint();
696      try {
697        UpdateResultsByTarget();
698      } finally { ResumeRepaint(true); }
699    }
700
701    private void generateTargetsButton_Click(object sender, EventArgs e) {
702      var maximization = IsMaximization();
703      decimal max = 1, min = 0, count = 10;
704      if (targets != null) {
705        max = (decimal)targets.Max();
706        min = (decimal)targets.Min();
707        count = targets.Length;
708      } else if (Content.Count > 0 && dataTableComboBox.SelectedIndex >= 0) {
709        var table = (string)dataTableComboBox.SelectedItem;
710        max = (decimal)Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Max(y => y.Item2)).Max();
711        min = (decimal)Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Min(y => y.Item2)).Min();
712        count = 6;
713      }
714      using (var dialog = new DefineArithmeticProgressionDialog(false, min, max, (max - min) / count)) {
715        if (dialog.ShowDialog() == DialogResult.OK) {
716          if (dialog.Values.Any()) {
717            targets = maximization ? dialog.Values.Select(x => (double)x).ToArray()
718                                  : dialog.Values.Reverse().Select(x => (double)x).ToArray();
719            suppressTargetsEvents = true;
720            targetsTextBox.Text = string.Join("% ; ", targets);
721            suppressTargetsEvents = false;
722
723            UpdateResultsByTarget();
724            SetEnabledStateOfControls();
725          }
726        }
727      }
728    }
729
730    private void addTargetsAsResultButton_Click(object sender, EventArgs e) {
731      var table = (string)dataTableComboBox.SelectedItem;
732      var maximization = IsMaximization();
733
734      var targetsPerProblem = (from r in Content
735                               let pd = new ProblemDescription(r).ToString()
736                               let target = r.Parameters.ContainsKey("BestKnownQuality")
737                                && r.Parameters["BestKnownQuality"] is DoubleValue
738                                  ? ((DoubleValue)r.Parameters["BestKnownQuality"]).Value
739                                  : ((IndexedDataTable<double>)r.Results[table]).Rows.First().Values.Last().Item2
740                               group target by pd into g
741                               select new { Problem = g.Key, Target = maximization ? g.Max() : g.Min() })
742                              .ToDictionary(x => x.Problem, x => x.Target);
743
744      foreach (var run in Content) {
745        if (!run.Results.ContainsKey(table)) continue;
746        var resultsTable = (IndexedDataTable<double>)run.Results[table];
747        var values = resultsTable.Rows.First().Values;
748        var i = 0;
749        var j = 0;
750        var pd = new ProblemDescription(run);
751        while (i < targets.Length && j < values.Count) {
752          var target = (maximization ? (1 - targets[i]) : (1 + targets[i])) * targetsPerProblem[pd.ToString()];
753          var current = values[j];
754          if (maximization && current.Item2 >= target
755              || !maximization && current.Item2 <= target) {
756            run.Results[table + ".Target" + target] = new DoubleValue(current.Item1);
757            i++;
758          } else {
759            j++;
760          }
761        }
762      }
763    }
764    #endregion
765
766    #region Event handlers for cost analysis
767    private bool suppressBudgetsEvents;
768    private void budgetsTextBox_Validating(object sender, CancelEventArgs e) {
769      if (suppressBudgetsEvents) return;
770      var budgetStrings = budgetsTextBox.Text.Split(new[] { ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
771      var budgetList = new List<double>();
772      foreach (var ts in budgetStrings) {
773        double b;
774        if (!double.TryParse(ts, out b)) {
775          errorProvider.SetError(budgetsTextBox, "Not all targets can be parsed: " + ts);
776          e.Cancel = true;
777          return;
778        }
779        budgetList.Add(b);
780      }
781      if (budgetList.Count == 0) {
782        errorProvider.SetError(budgetsTextBox, "Give at least one target value!");
783        e.Cancel = true;
784        return;
785      }
786      e.Cancel = false;
787      errorProvider.SetError(budgetsTextBox, null);
788      budgets = budgetList.ToArray();
789      UpdateResultsByCost();
790      SetEnabledStateOfControls();
791    }
792
793    private void eachOrAllBudgetsCheckBox_CheckedChanged(object sender, EventArgs e) {
794      var each = eachOrAllBudgetsCheckBox.Checked;
795      eachOrAllBudgetsCheckBox.Text = each ? "each" : "all";
796      SuspendRepaint();
797      try {
798        UpdateResultsByCost();
799      } finally { ResumeRepaint(true); }
800    }
801
802    private void generateBudgetsButton_Click(object sender, EventArgs e) {
803      decimal max = 1, min = 0, count = 10;
804      if (budgets != null) {
805        max = (decimal)budgets.Max();
806        min = (decimal)budgets.Min();
807        count = budgets.Length;
808      } else if (Content.Count > 0 && dataTableComboBox.SelectedIndex >= 0) {
809        var table = (string)dataTableComboBox.SelectedItem;
810        min = (decimal)Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Min(y => y.Item1)).Min();
811        max = (decimal)Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Max(y => y.Item1)).Max();
812        count = 6;
813      }
814      using (var dialog = new DefineArithmeticProgressionDialog(false, min, max, (max - min) / count)) {
815        if (dialog.ShowDialog() == DialogResult.OK) {
816          if (dialog.Values.Any()) {
817            budgets = dialog.Values.OrderBy(x => x).Select(x => (double)x).ToArray();
818
819            suppressBudgetsEvents = true;
820            budgetsTextBox.Text = string.Join(" ; ", budgets);
821            suppressBudgetsEvents = false;
822
823            UpdateResultsByCost();
824            SetEnabledStateOfControls();
825          }
826        }
827      }
828    }
829
830    private void addBudgetsAsResultButton_Click(object sender, EventArgs e) {
831      var table = (string)dataTableComboBox.SelectedItem;
832      var budgetStrings = budgetsTextBox.Text.Split(new[] { ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
833      if (budgetStrings.Length == 0) {
834        MessageBox.Show("Define a number of budgets.");
835        return;
836      }
837      var budgetList = new List<double>();
838      foreach (var bs in budgetStrings) {
839        double v;
840        if (!double.TryParse(bs, out v)) {
841          MessageBox.Show("Budgets must be a valid number: " + bs);
842          return;
843        }
844        budgetList.Add(v);
845      }
846      budgetList.Sort();
847
848      foreach (var run in Content) {
849        if (!run.Results.ContainsKey(table)) continue;
850        var resultsTable = (IndexedDataTable<double>)run.Results[table];
851        var values = resultsTable.Rows.First().Values;
852        var i = 0;
853        var j = 0;
854        Tuple<double, double> prev = null;
855        while (i < budgetList.Count && j < values.Count) {
856          var current = values[j];
857          if (current.Item1 >= budgetList[i]) {
858            if (prev != null || current.Item1 == budgetList[i]) {
859              var tgt = (prev == null || current.Item1 == budgetList[i]) ? current.Item2 : prev.Item2;
860              run.Results[table + ".Cost" + budgetList[i]] = new DoubleValue(tgt);
861            }
862            i++;
863          } else {
864            j++;
865            prev = current;
866          }
867        }
868      }
869    }
870    #endregion
871
872    #region Helpers
873    // Determines if the RunCollection contains maximization or minimization runs
874    private bool IsMaximization() {
875      if (Content == null) return false;
876      if (Content.Count > 0) {
877        foreach (var run in Content.Where(x => x.Parameters.ContainsKey("Maximization")
878                                            && x.Parameters["Maximization"] is BoolValue)) {
879          if (((BoolValue)run.Parameters["Maximization"]).Value) {
880            return true;
881          } else {
882            return false;
883          }
884        }
885        if (dataTableComboBox.SelectedIndex >= 0) {
886          var selectedTable = (string)dataTableComboBox.SelectedItem;
887          foreach (var run in Content.Where(x => x.Results.ContainsKey(selectedTable))) {
888            var table = run.Results[selectedTable] as IndexedDataTable<double>;
889            if (table == null) continue;
890            var firstRowValues = table.Rows.First().Values;
891            if (firstRowValues.Count < 2) continue;
892            return firstRowValues[0].Item2 < firstRowValues[firstRowValues.Count - 1].Item2;
893          }
894        }
895      }
896      // assume minimization by default
897      return false;
898    }
899    #endregion
900
901
902    private class ProblemDescription {
903      private readonly bool matchAll;
904      public static readonly ProblemDescription MatchAll = new ProblemDescription() {
905        ProblemName = "All with Best-Known"
906      };
907
908      private ProblemDescription() {
909        ProblemType = string.Empty;
910        ProblemName = string.Empty;
911        Evaluator = string.Empty;
912        DisplayProblemType = false;
913        DisplayProblemName = false;
914        DisplayEvaluator = false;
915        matchAll = true;
916      }
917
918      public ProblemDescription(IRun run) {
919        ProblemType = GetStringValueOrEmpty(run, "Problem Type");
920        ProblemName = GetStringValueOrEmpty(run, "Problem Name");
921        Evaluator = GetStringValueOrEmpty(run, "Evaluator");
922        DisplayProblemType = !string.IsNullOrEmpty(ProblemType);
923        DisplayProblemName = !string.IsNullOrEmpty(ProblemName);
924        DisplayEvaluator = !string.IsNullOrEmpty(Evaluator);
925        matchAll = false;
926      }
927
928      public bool DisplayProblemType { get; set; }
929      public string ProblemType { get; set; }
930      public bool DisplayProblemName { get; set; }
931      public string ProblemName { get; set; }
932      public bool DisplayEvaluator { get; set; }
933      public string Evaluator { get; set; }
934
935      public bool Match(IRun run) {
936        return matchAll ||
937               GetStringValueOrEmpty(run, "Problem Type") == ProblemType
938               && GetStringValueOrEmpty(run, "Problem Name") == ProblemName
939               && GetStringValueOrEmpty(run, "Evaluator") == Evaluator;
940      }
941
942      private string GetStringValueOrEmpty(IRun run, string key) {
943        return run.Parameters.ContainsKey(key) ? ((StringValue)run.Parameters[key]).Value : string.Empty;
944      }
945
946      public override bool Equals(object obj) {
947        var other = obj as ProblemDescription;
948        if (other == null) return false;
949        return ProblemType == other.ProblemType
950               && ProblemName == other.ProblemName
951               && Evaluator == other.Evaluator;
952      }
953
954      public override int GetHashCode() {
955        return ProblemType.GetHashCode() ^ ProblemName.GetHashCode() ^ Evaluator.GetHashCode();
956      }
957
958      public override string ToString() {
959        return string.Join("  --  ", new[] {
960          (DisplayProblemType ? ProblemType : string.Empty),
961          (DisplayProblemName ? ProblemName : string.Empty),
962          (DisplayEvaluator ? Evaluator : string.Empty) }.Where(x => !string.IsNullOrEmpty(x)));
963      }
964    }
965  }
966}
Note: See TracBrowser for help on using the repository browser.