Free cookie consent management tool by TermsFeed Policy Generator

source: branches/2520_PersistenceReintegration/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs @ 16724

Last change on this file since 16724 was 16453, checked in by jkarder, 6 years ago

#2520: updated year of copyrights

File size: 57.9 KB
RevLine 
[7980]1#region License Information
2/* HeuristicLab
[16453]3 * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
[7980]4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
[11344]24using System.ComponentModel;
[12822]25using System.Drawing;
[12841]26using System.Globalization;
[7980]27using System.Linq;
[12771]28using System.Windows.Forms;
[14101]29using System.Windows.Forms.DataVisualization.Charting;
30using HeuristicLab.Analysis;
31using HeuristicLab.Collections;
[14647]32using HeuristicLab.Core;
[14101]33using HeuristicLab.Core.Views;
34using HeuristicLab.Data;
35using HeuristicLab.MainForm;
36using HeuristicLab.MainForm.WindowsForms;
[7980]37
38namespace HeuristicLab.Optimization.Views {
[12803]39  [View("Run-length Distribution View")]
[7980]40  [Content(typeof(RunCollection), false)]
[12803]41  public partial class RunCollectionRLDView : ItemView {
[14101]42    private List<Series> invisibleTargetSeries;
43
[14775]44    private const string AllInstances = "All Instances";
[7980]45
[12822]46    private static readonly Color[] colors = new[] {
[12838]47      Color.FromArgb(0x40, 0x6A, 0xB7),
48      Color.FromArgb(0xB1, 0x6D, 0x01),
49      Color.FromArgb(0x4E, 0x8A, 0x06),
[12822]50      Color.FromArgb(0x75, 0x50, 0x7B),
51      Color.FromArgb(0x72, 0x9F, 0xCF),
52      Color.FromArgb(0xA4, 0x00, 0x00),
53      Color.FromArgb(0xAD, 0x7F, 0xA8),
[12838]54      Color.FromArgb(0x29, 0x50, 0xCF),
55      Color.FromArgb(0x90, 0xB0, 0x60),
56      Color.FromArgb(0xF5, 0x89, 0x30),
[12822]57      Color.FromArgb(0x55, 0x57, 0x53),
[12838]58      Color.FromArgb(0xEF, 0x59, 0x59),
59      Color.FromArgb(0xED, 0xD4, 0x30),
60      Color.FromArgb(0x63, 0xC2, 0x16),
[12822]61    };
[14101]62    private static readonly ChartDashStyle[] lineStyles = new[] {
63      ChartDashStyle.Solid,
64      ChartDashStyle.Dash,
65      ChartDashStyle.DashDot,
66      ChartDashStyle.Dot
67    };
68    private static readonly DataRowVisualProperties.DataRowLineStyle[] hlLineStyles = new[] {
[12838]69      DataRowVisualProperties.DataRowLineStyle.Solid,
70      DataRowVisualProperties.DataRowLineStyle.Dash,
71      DataRowVisualProperties.DataRowLineStyle.DashDot,
72      DataRowVisualProperties.DataRowLineStyle.Dot
73    };
[12822]74
[7980]75    public new RunCollection Content {
76      get { return (RunCollection)base.Content; }
77      set { base.Content = value; }
78    }
79
[12838]80    private double[] targets;
81    private double[] budgets;
[14647]82    private bool targetsAreRelative = true;
[15125]83    private bool showLabelsInTargetChart = true;
[14647]84    private readonly BindingList<ProblemInstance> problems;
[12804]85
[7980]86    private bool suppressUpdates;
[12838]87    private readonly IndexedDataTable<double> byCostDataTable;
88    public IndexedDataTable<double> ByCostDataTable {
89      get { return byCostDataTable; }
90    }
[7980]91
[12803]92    public RunCollectionRLDView() {
[8108]93      InitializeComponent();
[14101]94      invisibleTargetSeries = new List<Series>();
95
96      targetChart.CustomizeAllChartAreas();
97      targetChart.ChartAreas[0].CursorX.Interval = 1;
98      targetChart.SuppressExceptions = true;
[15125]99      byCostDataTable = new IndexedDataTable<double>("ECDF by Cost", "A data table containing the ECDF of function values (relative to best-known).") {
[12838]100        VisualProperties = {
[15125]101          YAxisTitle = "Proportion of runs",
[13583]102          YAxisMinimumFixedValue = 0,
103          YAxisMinimumAuto = false,
104          YAxisMaximumFixedValue = 1,
105          YAxisMaximumAuto = false
[12838]106        }
107      };
108      byCostViewHost.Content = byCostDataTable;
[14652]109      suppressUpdates = true;
110      relativeOrAbsoluteComboBox.SelectedItem = targetsAreRelative ? "relative" : "absolute";
[14647]111      problems = new BindingList<ProblemInstance>();
112      problemComboBox.DataSource = new BindingSource() { DataSource = problems };
113      problemComboBox.DataBindings.DefaultDataSourceUpdateMode = DataSourceUpdateMode.OnPropertyChanged;
[8108]114      suppressUpdates = false;
115    }
116
[7980]117    #region Content events
118    protected override void RegisterContentEvents() {
119      base.RegisterContentEvents();
[12631]120      Content.ItemsAdded += Content_ItemsAdded;
121      Content.ItemsRemoved += Content_ItemsRemoved;
122      Content.CollectionReset += Content_CollectionReset;
123      Content.UpdateOfRunsInProgressChanged += Content_UpdateOfRunsInProgressChanged;
124      Content.OptimizerNameChanged += Content_AlgorithmNameChanged;
[7980]125    }
126    protected override void DeregisterContentEvents() {
[12631]127      Content.ItemsAdded -= Content_ItemsAdded;
128      Content.ItemsRemoved -= Content_ItemsRemoved;
129      Content.CollectionReset -= Content_CollectionReset;
130      Content.UpdateOfRunsInProgressChanged -= Content_UpdateOfRunsInProgressChanged;
131      Content.OptimizerNameChanged -= Content_AlgorithmNameChanged;
[7980]132      base.DeregisterContentEvents();
133    }
134
135    private void Content_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IRun> e) {
[12631]136      if (suppressUpdates) return;
[7980]137      if (InvokeRequired) {
138        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_ItemsAdded), sender, e);
139        return;
140      }
[12841]141      UpdateGroupAndProblemComboBox();
[12631]142      UpdateDataTableComboBox();
[12771]143      foreach (var run in e.Items)
144        RegisterRunEvents(run);
[7980]145    }
146    private void Content_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IRun> e) {
[12631]147      if (suppressUpdates) return;
[7980]148      if (InvokeRequired) {
149        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_ItemsRemoved), sender, e);
150        return;
151      }
[12841]152      UpdateGroupAndProblemComboBox();
[12631]153      UpdateDataTableComboBox();
[12771]154      foreach (var run in e.Items)
155        DeregisterRunEvents(run);
[7980]156    }
157    private void Content_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
[12631]158      if (suppressUpdates) return;
[7980]159      if (InvokeRequired) {
160        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset), sender, e);
161        return;
162      }
[12841]163      UpdateGroupAndProblemComboBox();
[12631]164      UpdateDataTableComboBox();
[12771]165      foreach (var run in e.OldItems)
166        DeregisterRunEvents(run);
[7980]167    }
[8738]168    private void Content_AlgorithmNameChanged(object sender, EventArgs e) {
169      if (InvokeRequired)
170        Invoke(new EventHandler(Content_AlgorithmNameChanged), sender, e);
171      else UpdateCaption();
172    }
[7980]173    private void Content_UpdateOfRunsInProgressChanged(object sender, EventArgs e) {
174      if (InvokeRequired) {
175        Invoke(new EventHandler(Content_UpdateOfRunsInProgressChanged), sender, e);
176        return;
177      }
178      suppressUpdates = Content.UpdateOfRunsInProgress;
[12631]179      if (!suppressUpdates) {
180        UpdateDataTableComboBox();
[12841]181        UpdateGroupAndProblemComboBox();
[12771]182        UpdateRuns();
[12631]183      }
[7980]184    }
185
186    private void RegisterRunEvents(IRun run) {
[11344]187      run.PropertyChanged += run_PropertyChanged;
[7980]188    }
189    private void DeregisterRunEvents(IRun run) {
[11344]190      run.PropertyChanged -= run_PropertyChanged;
[7980]191    }
[11344]192    private void run_PropertyChanged(object sender, PropertyChangedEventArgs e) {
[7980]193      if (suppressUpdates) return;
[11344]194      if (InvokeRequired) {
195        Invoke((Action<object, PropertyChangedEventArgs>)run_PropertyChanged, sender, e);
196      } else {
[12771]197        if (e.PropertyName == "Visible")
198          UpdateRuns();
[11344]199      }
[7980]200    }
201    #endregion
202
203    protected override void OnContentChanged() {
204      base.OnContentChanged();
205      dataTableComboBox.Items.Clear();
[12771]206      groupComboBox.Items.Clear();
[14101]207      targetChart.ChartAreas[0].AxisX.IsLogarithmic = false;
208      targetChart.Series.Clear();
209      invisibleTargetSeries.Clear();
210      byCostDataTable.VisualProperties.XAxisLogScale = false;
211      byCostDataTable.Rows.Clear();
[7980]212
[8738]213      UpdateCaption();
[7980]214      if (Content != null) {
[12841]215        UpdateGroupAndProblemComboBox();
[7980]216        UpdateDataTableComboBox();
217      }
218    }
219
[12806]220
[12841]221    private void UpdateGroupAndProblemComboBox() {
222      var selectedGroupItem = (string)groupComboBox.SelectedItem;
[12599]223
[12774]224      var groupings = Content.ParameterNames.OrderBy(x => x).ToArray();
[12771]225      groupComboBox.Items.Clear();
[14775]226      groupComboBox.Items.Add(AllInstances);
[12771]227      groupComboBox.Items.AddRange(groupings);
[12841]228      if (selectedGroupItem != null && groupComboBox.Items.Contains(selectedGroupItem)) {
229        groupComboBox.SelectedItem = selectedGroupItem;
[12771]230      } else if (groupComboBox.Items.Count > 0) {
231        groupComboBox.SelectedItem = groupComboBox.Items[0];
[7980]232      }
[12841]233
[14647]234      var table = (string)dataTableComboBox.SelectedItem;
235      var problemDict = CalculateBestTargetPerProblemInstance(table);
[12841]236
[14647]237      var problemTypesDifferent = problemDict.Keys.Select(x => x.ProblemType).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
238      var problemNamesDifferent = problemDict.Keys.Select(x => x.ProblemName).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
239      var evaluatorDifferent = problemDict.Keys.Select(x => x.Evaluator).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
240      var maximizationDifferent = problemDict.Keys.Select(x => x.Maximization).Distinct().Count() > 1;
[13583]241      var allEqual = !problemTypesDifferent && !problemNamesDifferent && !evaluatorDifferent && !maximizationDifferent;
[12841]242
[14647]243      var selectedProblemItem = (ProblemInstance)problemComboBox.SelectedItem;
[14775]244      problemComboBox.DataSource = null;
245      problemComboBox.Items.Clear();
[14647]246      problems.Clear();
247      problems.Add(ProblemInstance.MatchAll);
[14775]248      problemComboBox.DataSource = new BindingSource() { DataSource = problems };
249      problemComboBox.DataBindings.DefaultDataSourceUpdateMode = DataSourceUpdateMode.OnPropertyChanged;
[14647]250      if (problems[0].Equals(selectedProblemItem)) problemComboBox.SelectedItem = problems[0];
251      foreach (var p in problemDict.ToList()) {
252        p.Key.BestKnownQuality = p.Value;
253        p.Key.DisplayProblemType = problemTypesDifferent;
254        p.Key.DisplayProblemName = problemNamesDifferent || allEqual;
255        p.Key.DisplayEvaluator = evaluatorDifferent;
256        p.Key.DisplayMaximization = maximizationDifferent;
257        problems.Add(p.Key);
258        if (p.Key.Equals(selectedProblemItem)) problemComboBox.SelectedItem = p.Key;
[12841]259      }
260      SetEnabledStateOfControls();
[7980]261    }
262
263    private void UpdateDataTableComboBox() {
[12631]264      string selectedItem = (string)dataTableComboBox.SelectedItem;
265
[7986]266      dataTableComboBox.Items.Clear();
[7980]267      var dataTables = (from run in Content
268                        from result in run.Results
[12771]269                        where result.Value is IndexedDataTable<double>
[7980]270                        select result.Key).Distinct().ToArray();
271
272      dataTableComboBox.Items.AddRange(dataTables);
[12631]273      if (selectedItem != null && dataTableComboBox.Items.Contains(selectedItem)) {
274        dataTableComboBox.SelectedItem = selectedItem;
275      } else if (dataTableComboBox.Items.Count > 0) {
276        dataTableComboBox.SelectedItem = dataTableComboBox.Items[0];
277      }
[7980]278    }
279
[12838]280    protected override void SetEnabledStateOfControls() {
281      base.SetEnabledStateOfControls();
282      groupComboBox.Enabled = Content != null;
[12841]283      problemComboBox.Enabled = Content != null && problemComboBox.Items.Count > 1;
284      dataTableComboBox.Enabled = Content != null && dataTableComboBox.Items.Count > 1;
[12838]285      addTargetsAsResultButton.Enabled = Content != null && targets != null && dataTableComboBox.SelectedIndex >= 0;
286      addBudgetsAsResultButton.Enabled = Content != null && budgets != null && dataTableComboBox.SelectedIndex >= 0;
[15125]287      generateTargetsButton.Enabled = targets != null;
[12838]288    }
289
[14775]290    private IEnumerable<AlgorithmInstance> GroupRuns() {
[12841]291      var table = (string)dataTableComboBox.SelectedItem;
[14775]292      if (string.IsNullOrEmpty(table)) yield break;
[12841]293
294      var selectedGroup = (string)groupComboBox.SelectedItem;
[14775]295      if (string.IsNullOrEmpty(selectedGroup)) yield break;
[12841]296
[14647]297      var selectedProblem = (ProblemInstance)problemComboBox.SelectedItem;
[14775]298      if (selectedProblem == null) yield break;
[14647]299     
[12864]300      foreach (var x in (from r in Content
[14775]301                         where (selectedGroup == AllInstances || r.Parameters.ContainsKey(selectedGroup))
[12864]302                           && selectedProblem.Match(r)
303                           && r.Results.ContainsKey(table)
304                           && r.Visible
[14775]305                         let key = selectedGroup == AllInstances ? AllInstances : r.Parameters[selectedGroup].ToString()
[14654]306                         group r by key into g
[14775]307                         select new AlgorithmInstance(g.Key, g, problems))) {
308        yield return x;
[12864]309      }
[12841]310    }
311
[12838]312    #region Performance analysis by (multiple) target(s)
313    private void UpdateResultsByTarget() {
314      // necessary to reset log scale -> empty chart cannot use log scaling
[14101]315      targetChart.ChartAreas[0].AxisX.IsLogarithmic = false;
316      targetChart.Series.Clear();
317      invisibleTargetSeries.Clear();
[15125]318     
[12838]319      var table = (string)dataTableComboBox.SelectedItem;
320      if (string.IsNullOrEmpty(table)) return;
321
[12864]322      if (targets == null) GenerateDefaultTargets();
[12838]323
[14775]324      var algInstances = GroupRuns().ToList();
325      if (algInstances.Count == 0) return;
[12838]326
327      var xAxisTitles = new HashSet<string>();
328
[14654]329      // hits describes the number of target hits at a certain time for a certain group
330      var hits = new Dictionary<string, SortedList<double, int>>();
331      // misses describes the number of target misses after a certain time for a certain group
332      // for instance when a run ends, but has not achieved all targets, misses describes
333      // how many targets have been left open at the point when the run ended
334      var misses = new Dictionary<string, SortedList<double, int>>();
[14775]335      var totalRuns = new Dictionary<string, int>();
[14654]336
337      var aggregate = aggregateTargetsCheckBox.Checked;
[14101]338      double minEff = double.MaxValue, maxEff = double.MinValue;
[14775]339      foreach (var alg in algInstances) {
340        var noRuns = 0;
[14654]341        SortedList<double, int> epdfHits = null, epdfMisses = null;
342        if (aggregate) {
[14775]343          hits[alg.Name] = epdfHits = new SortedList<double, int>();
344          misses[alg.Name] = epdfMisses = new SortedList<double, int>();
[14654]345        }
[14775]346        foreach (var problem in alg.GetProblemInstances()) {
347          var max = problem.IsMaximization();
348          var absTargets = GetAbsoluteTargets(problem).ToArray();
349          foreach (var run in alg.GetRuns(problem)) {
[14654]350            noRuns++;
351            var resultsTable = (IndexedDataTable<double>)run.Results[table];
352            xAxisTitles.Add(resultsTable.VisualProperties.XAxisTitle);
353
354            var efforts = absTargets.Select(t => GetEffortToHitTarget(resultsTable.Rows.First().Values, t, max)).ToArray();
355            minEff = Math.Min(minEff, efforts.Min(x => x.Item2));
356            maxEff = Math.Max(maxEff, efforts.Max(x => x.Item2));
357            for (var idx = 0; idx < efforts.Length; idx++) {
358              var e = efforts[idx];
359              if (!aggregate) {
[14775]360                var key = alg.Name + "@" + (targetsAreRelative
361                            ? (targets[idx] * 100).ToString(CultureInfo.CurrentCulture.NumberFormat) + "%"
362                            : targets[idx].ToString(CultureInfo.CurrentCulture.NumberFormat));
[14654]363                if (!hits.TryGetValue(key, out epdfHits))
364                  hits[key] = epdfHits = new SortedList<double, int>();
365                if (!misses.TryGetValue(key, out epdfMisses))
366                  misses[key] = epdfMisses = new SortedList<double, int>();
[14775]367                totalRuns[key] = noRuns;
368              };
[14654]369              var list = e.Item1 ? epdfHits : epdfMisses;
370              int v;
371              if (list.TryGetValue(e.Item2, out v))
372                list[e.Item2] = v + 1;
373              else list[e.Item2] = 1;
374            }
[14035]375          }
376        }
[14775]377        if (aggregate) totalRuns[alg.Name] = noRuns;
[14035]378      }
379
[14654]380      UpdateTargetChartAxisXBounds(minEff, maxEff);
[14101]381
[14775]382      DrawTargetsEcdf(hits, misses, totalRuns);
[12838]383
[14654]384      if (targets.Length == 1) {
385        if (targetsAreRelative)
386          targetChart.ChartAreas[0].AxisY.Title = "Probability to be " + (targets[0] * 100) + "% worse than best";
387        else targetChart.ChartAreas[0].AxisY.Title = "Probability to reach at least a fitness of " + targets[0];
388      } else targetChart.ChartAreas[0].AxisY.Title = "Proportion of reached targets";
389      targetChart.ChartAreas[0].AxisX.Title = string.Join(" / ", xAxisTitles);
390      targetChart.ChartAreas[0].AxisX.IsLogarithmic = CanDisplayLogarithmic();
391      targetChart.ChartAreas[0].CursorY.Interval = 0.05;
[12838]392
[14775]393      UpdateErtTables(algInstances);
[14654]394    }
[12838]395
[14775]396    private void DrawTargetsEcdf(Dictionary<string, SortedList<double, int>> hits, Dictionary<string, SortedList<double, int>> misses, Dictionary<string, int> noRuns) {
[14654]397      var colorCount = 0;
398      var lineStyleCount = 0;
399     
400      var showMarkers = markerCheckBox.Checked;
401      foreach (var list in hits) {
402        var row = new Series(list.Key) {
403          ChartType = SeriesChartType.StepLine,
404          BorderWidth = 3,
405          Color = colors[colorCount],
406          BorderDashStyle = lineStyles[lineStyleCount],
407        };
408        var rowShade = new Series(list.Key + "-range") {
409          IsVisibleInLegend = false,
410          ChartType = SeriesChartType.Range,
411          Color = Color.FromArgb(32, colors[colorCount]),
412          YValuesPerPoint = 2
413        };
414
415        var ecdf = 0.0;
416        var missedecdf = 0.0;
417        var iter = misses[list.Key].GetEnumerator();
418        var moreMisses = iter.MoveNext();
[14775]419        var totalTargets = noRuns[list.Key];
[14654]420        if (aggregateTargetsCheckBox.Checked) totalTargets *= targets.Length;
421        var movingTargets = totalTargets;
422        var labelPrinted = false;
423        foreach (var h in list.Value) {
424          var prevmissedecdf = missedecdf;
425          while (moreMisses && iter.Current.Key <= h.Key) {
426            if (!labelPrinted && row.Points.Count > 0) {
427              var point = row.Points.Last();
[15125]428              if (showLabelsInTargetChart)
429                point.Label = row.Name;
[14654]430              point.MarkerStyle = MarkerStyle.Cross;
431              point.MarkerBorderWidth = 1;
432              point.MarkerSize = 10;
433              labelPrinted = true;
434              rowShade.Points.Add(new DataPoint(point.XValue, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
[14101]435            }
436            missedecdf += iter.Current.Value;
[14654]437            movingTargets -= iter.Current.Value;
[14652]438            if (row.Points.Count > 0 && row.Points.Last().XValue == iter.Current.Key) {
[14101]439              row.Points.Last().SetValueY(ecdf / movingTargets);
[14654]440              row.Points.Last().Color = Color.FromArgb(255 - (int)Math.Floor(255 * (prevmissedecdf / totalTargets)), colors[colorCount]);
[14652]441            } else {
442              var dp = new DataPoint(iter.Current.Key, ecdf / movingTargets) {
[14654]443                Color = Color.FromArgb(255 - (int)Math.Floor(255 * (prevmissedecdf / totalTargets)), colors[colorCount])
[14652]444              };
445              if (showMarkers) {
446                dp.MarkerStyle = MarkerStyle.Circle;
447                dp.MarkerBorderWidth = 1;
448                dp.MarkerSize = 5;
449              }
450              row.Points.Add(dp);
[14654]451              prevmissedecdf = missedecdf;
[14652]452            }
[14101]453            if (boundShadingCheckBox.Checked) {
454              if (rowShade.Points.Count > 0 && rowShade.Points.Last().XValue == iter.Current.Key)
455                rowShade.Points.Last().SetValueY(ecdf / totalTargets, (ecdf + missedecdf) / totalTargets);
[14654]456              else rowShade.Points.Add(new DataPoint(iter.Current.Key, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
[14101]457            }
458            moreMisses = iter.MoveNext();
[14654]459            if (!labelPrinted) {
460              var point = row.Points.Last();
[15125]461              if (showLabelsInTargetChart)
462                point.Label = row.Name;
[14654]463              point.MarkerStyle = MarkerStyle.Cross;
464              point.MarkerBorderWidth = 1;
465              point.MarkerSize = 10;
466              labelPrinted = true;
467            }
[14101]468          }
[14654]469          ecdf += h.Value;
470          if (row.Points.Count > 0 && row.Points.Last().XValue == h.Key) {
471            row.Points.Last().SetValueY(ecdf / movingTargets);
472            row.Points.Last().Color = Color.FromArgb(255 - (int)Math.Floor(255 * (missedecdf / totalTargets)), colors[colorCount]);
473          } else {
474            var dp = new DataPoint(h.Key, ecdf / movingTargets) {
[14652]475              Color = Color.FromArgb(255 - (int)Math.Floor(255 * (missedecdf / totalTargets)), colors[colorCount])
476            };
477            if (showMarkers) {
478              dp.MarkerStyle = MarkerStyle.Circle;
479              dp.MarkerBorderWidth = 1;
480              dp.MarkerSize = 5;
481            }
482            row.Points.Add(dp);
[14101]483          }
[14654]484          if (missedecdf > 0 && boundShadingCheckBox.Checked) {
485            if (rowShade.Points.Count > 0 && rowShade.Points.Last().XValue == h.Key)
486              rowShade.Points.Last().SetValueY(ecdf / totalTargets, (ecdf + missedecdf) / totalTargets);
487            else rowShade.Points.Add(new DataPoint(h.Key, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
488          }
489        }
[14101]490
[14654]491        while (moreMisses) {
492          // if there are misses beyond the last hit we extend the shaded area
493          missedecdf += iter.Current.Value;
494          var dp = new DataPoint(iter.Current.Key, ecdf / movingTargets) {
495            Color = Color.FromArgb(255 - (int)Math.Floor(255 * (missedecdf / totalTargets)), colors[colorCount])
496          };
497          if (showMarkers) {
498            dp.MarkerStyle = MarkerStyle.Circle;
499            dp.MarkerBorderWidth = 1;
500            dp.MarkerSize = 5;
501          }
502          row.Points.Add(dp);
503          if (boundShadingCheckBox.Checked) {
504            rowShade.Points.Add(new DataPoint(iter.Current.Key, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
505          }
506          moreMisses = iter.MoveNext();
507
508          if (!labelPrinted && row.Points.Count > 0) {
509            var point = row.Points.Last();
[15125]510            if (showLabelsInTargetChart)
511              point.Label = row.Name;
[14654]512            point.MarkerStyle = MarkerStyle.Cross;
513            point.MarkerBorderWidth = 1;
514            point.MarkerSize = 10;
515            labelPrinted = true;
516          }
[12838]517        }
[14654]518
519        if (!labelPrinted) {
520          var point = row.Points.Last();
[15125]521          if (showLabelsInTargetChart)
522            point.Label = row.Name;
[14654]523          point.MarkerStyle = MarkerStyle.Cross;
524          point.MarkerBorderWidth = 1;
525          point.MarkerSize = 10;
526          rowShade.Points.Add(new DataPoint(point.XValue, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
527          labelPrinted = true;
528        }
529
530        ConfigureSeries(row);
531        targetChart.Series.Add(rowShade);
532        targetChart.Series.Add(row);
533
[12838]534        colorCount = (colorCount + 1) % colors.Length;
535        if (colorCount == 0) lineStyleCount = (lineStyleCount + 1) % lineStyles.Length;
536      }
[14654]537    }
[12838]538
[14654]539    private void UpdateTargetChartAxisXBounds(double minEff, double maxEff) {
540      var minZeros = (int)Math.Floor(Math.Log10(minEff));
541      var maxZeros = (int)Math.Floor(Math.Log10(maxEff));
542      var axisMin = (decimal)Math.Pow(10, minZeros);
543      var axisMax = (decimal)Math.Pow(10, maxZeros);
544      if (!targetLogScalingCheckBox.Checked) {
545        var minAdd = (decimal)Math.Pow(10, minZeros - 1) * 2;
546        var maxAdd = (decimal)Math.Pow(10, maxZeros - 1) * 2;
547        while (axisMin + minAdd < (decimal)minEff) axisMin += minAdd;
548        while (axisMax <= (decimal)maxEff) axisMax += maxAdd;
549      } else axisMax = (decimal)Math.Pow(10, (int)Math.Ceiling(Math.Log10(maxEff)));
550      targetChart.ChartAreas[0].AxisX.Minimum = (double)axisMin;
551      targetChart.ChartAreas[0].AxisX.Maximum = (double)axisMax;
[12838]552    }
[14654]553
554    private IEnumerable<double> GetAbsoluteTargets(ProblemInstance pInstance) {
[14647]555      if (!targetsAreRelative) return targets;
[14654]556      var maximization = pInstance.IsMaximization();
557      var bestKnown = pInstance.BestKnownQuality;
558      if (double.IsNaN(bestKnown)) throw new ArgumentException("Problem instance does not have a defined best - known quality.");
[14647]559      IEnumerable<double> tmp = null;
560      if (bestKnown > 0) {
561        tmp = targets.Select(x => (maximization ? (1 - x) : (1 + x)) * bestKnown);
562      } else if (bestKnown < 0) {
563        tmp = targets.Select(x => (!maximization ? (1 - x) : (1 + x)) * bestKnown);
564      } else {
565        // relative to 0 is impossible
566        tmp = targets;
567      }
568      return tmp;
569    }
[12838]570
[14654]571    private double[] GetAbsoluteTargetsWorstToBest(ProblemInstance pInstance) {
572      if (double.IsNaN(pInstance.BestKnownQuality)) throw new ArgumentException("Problem instance does not have a defined best-known quality.");
573      var absTargets = GetAbsoluteTargets(pInstance);
574      return (pInstance.IsMaximization()
575        ? absTargets.OrderBy(x => x) : absTargets.OrderByDescending(x => x)).ToArray();
[14647]576    }
577
[12864]578    private void GenerateDefaultTargets() {
[14654]579      targets = new[] { 0.1, 0.095, 0.09, 0.085, 0.08, 0.075, 0.07, 0.065, 0.06, 0.055, 0.05, 0.045, 0.04, 0.035, 0.03, 0.025, 0.02, 0.015, 0.01, 0.005, 0 };
[15125]580      SynchronizeTargetTextBox();
[12838]581    }
582
[14654]583    private Tuple<bool, double> GetEffortToHitTarget(
[15125]584        ObservableList<Tuple<double, double>> convergenceGraph,
[14654]585        double absTarget, bool maximization) {
[15125]586      if (convergenceGraph.Count == 0)
587        throw new ArgumentException("Convergence graph is empty.", "convergenceGraph");
588     
589      var index = convergenceGraph.BinarySearch(Tuple.Create(0.0, absTarget), new TargetComparer(maximization));
590      if (index >= 0) {
591        return Tuple.Create(true, convergenceGraph[index].Item1);
592      } else {
593        index = ~index;
594        if (index >= convergenceGraph.Count)
595          return Tuple.Create(false, convergenceGraph.Last().Item1);
596        return Tuple.Create(true, convergenceGraph[index].Item1);
[12838]597      }
598    }
599
[14775]600    private void UpdateErtTables(List<AlgorithmInstance> algorithmInstances) {
[12888]601      ertTableView.Content = null;
[15125]602      var columns = targets.Length + 1;
603      var totalRows = algorithmInstances.Count * algorithmInstances.Max(x => x.GetNumberOfProblemInstances()) + algorithmInstances.Max(x => x.GetNumberOfProblemInstances());
604      var matrix = new StringMatrix(totalRows, columns);
605      var rowNames = new List<string>();
606      matrix.ColumnNames = targets.Select(x => targetsAreRelative ? (100 * x).ToString() + "%" : x.ToString())
607        .Concat(new[] { "#succ" }).ToList();
[12841]608      var rowCount = 0;
609
[12956]610      var tableName = (string)dataTableComboBox.SelectedItem;
611      if (string.IsNullOrEmpty(tableName)) return;
[14647]612     
[14775]613      var problems = algorithmInstances.SelectMany(x => x.GetProblemInstances()).Distinct().ToList();
[12841]614
[12864]615      foreach (var problem in problems) {
[14647]616        var max = problem.IsMaximization();
[15125]617        rowNames.Add(problem.ToString());
[14654]618        var absTargets = GetAbsoluteTargetsWorstToBest(problem);
[15125]619        if (targetsAreRelative) {
620          // print out the absolute target values
621          for (var i = 0; i < absTargets.Length; i++) {
622            matrix[rowCount, i] = absTargets[i].ToString("##,0.0", CultureInfo.CurrentCulture.NumberFormat);
623          }
[12841]624        }
625        rowCount++;
[12864]626
[14775]627        foreach (var alg in algorithmInstances) {
[15125]628          rowNames.Add(alg.Name);
[14775]629          var runs = alg.GetRuns(problem).ToList();
630          if (runs.Count == 0) {
[12939]631            matrix[rowCount, columns - 1] = "N/A";
632            rowCount++;
633            continue;
634          }
[14647]635          var result = default(ErtCalculationResult);
636          for (var i = 0; i < absTargets.Length; i++) {
637            result = ExpectedRuntimeHelper.CalculateErt(runs, tableName, absTargets[i], max);
[15125]638            matrix[rowCount, i] = result.ToString();
[12864]639          }
[12956]640          matrix[rowCount, columns - 1] = targets.Length > 0 ? result.SuccessfulRuns + "/" + result.TotalRuns : "-";
[12864]641          rowCount++;
642        }
[12841]643      }
[15125]644      matrix.RowNames = rowNames;
645      ertTableView.Content = matrix;
[12888]646      ertTableView.DataGridView.AutoResizeColumns(DataGridViewAutoSizeColumnsMode.AllCells);
[12841]647    }
[12838]648    #endregion
649
650    #region Performance analysis by (multiple) budget(s)
651    private void UpdateResultsByCost() {
652      // necessary to reset log scale -> empty chart cannot use log scaling
653      byCostDataTable.VisualProperties.XAxisLogScale = false;
654      byCostDataTable.Rows.Clear();
655
656      var table = (string)dataTableComboBox.SelectedItem;
657      if (string.IsNullOrEmpty(table)) return;
658
659      if (budgets == null) GenerateDefaultBudgets(table);
660
[14775]661      var algInstances = GroupRuns().ToList();
662      if (algInstances.Count == 0) return;
[12838]663
664      var colorCount = 0;
665      var lineStyleCount = 0;
[14647]666     
[14775]667      foreach (var alg in algInstances) {
[12838]668        var hits = new Dictionary<string, SortedList<double, double>>();
669
[14775]670        foreach (var problem in alg.GetProblemInstances()) {
671          foreach (var run in alg.GetRuns(problem)) {
[12864]672            var resultsTable = (IndexedDataTable<double>)run.Results[table];
[12838]673
[15125]674            CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), problem, alg.Name);
[12838]675          }
676        }
677
678        foreach (var list in hits) {
679          var row = new IndexedDataRow<double>(list.Key) {
680            VisualProperties = {
681              ChartType = DataRowVisualProperties.DataRowChartType.StepLine,
682              LineWidth = 2,
683              Color = colors[colorCount],
[14101]684              LineStyle = hlLineStyles[lineStyleCount],
[13745]685              StartIndexZero = false
[12838]686            }
687          };
688
689          var total = 0.0;
[15125]690          var count = list.Value.Count;
[12838]691          foreach (var h in list.Value) {
692            total += h.Value;
[15125]693            row.Values.Add(Tuple.Create(h.Key, total / (double)count));
[12838]694          }
695
696          byCostDataTable.Rows.Add(row);
697        }
698        colorCount = (colorCount + 1) % colors.Length;
699        if (colorCount == 0) lineStyleCount = (lineStyleCount + 1) % lineStyles.Length;
700      }
701
[12865]702      byCostDataTable.VisualProperties.XAxisTitle = "Targets to Best-Known Ratio";
[12838]703      byCostDataTable.VisualProperties.XAxisLogScale = byCostDataTable.Rows.Count > 0 && budgetLogScalingCheckBox.Checked;
704    }
705
706    private void GenerateDefaultBudgets(string table) {
[14775]707      var runs = Content;
[12841]708      var min = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Min()).Min();
709      var max = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Max()).Max();
[15125]710      var points = 3;
[14654]711      budgets = Enumerable.Range(1, points).Select(x => min + (x / (double)points) * (max - min)).ToArray();
[12838]712      suppressBudgetsEvents = true;
713      budgetsTextBox.Text = string.Join(" ; ", budgets);
714      suppressBudgetsEvents = false;
715    }
716
[15125]717    private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, ProblemInstance problem, string groupName) {
[14647]718      var max = problem.IsMaximization();
[15125]719      var prevIndex = 0;
[12838]720      foreach (var b in budgets) {
721        var key = groupName + "-" + b;
[15125]722        var index = row.Values.BinarySearch(prevIndex, row.Values.Count - prevIndex, Tuple.Create(b, 0.0), new CostComparer());
723        if (index < 0) {
724          index = ~index;
725          if (index >= row.Values.Count) break; // the run wasn't long enough to use up budget b (or any subsequent larger one)
726        }
[12838]727        if (!hits.ContainsKey(key)) hits.Add(key, new SortedList<double, double>());
[15125]728        var v = row.Values[index];
729        var relTgt = CalculateRelativeDifference(max, problem.BestKnownQuality, v.Item2) + 1;
730        if (hits[key].ContainsKey(relTgt))
731          hits[key][relTgt]++;
732        else hits[key][relTgt] = 1.0;
733        prevIndex = index;
[12838]734      }
735    }
736    #endregion
737
[8738]738    private void UpdateCaption() {
[12803]739      Caption = Content != null ? Content.OptimizerName + " RLD View" : ViewAttribute.GetViewName(GetType());
[8738]740    }
741
[15125]742    private void SynchronizeTargetTextBox() {
743      if (InvokeRequired) Invoke((Action)SynchronizeTargetTextBox);
744      else {
745        suppressTargetsEvents = true;
746        try {
747          if (targetsAreRelative)
748            targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100)) + "%";
749          else targetsTextBox.Text = string.Join(" ; ", targets);
750        } finally { suppressTargetsEvents = false; }
751      }
752    }
753
[12771]754    private void groupComboBox_SelectedIndexChanged(object sender, EventArgs e) {
755      UpdateRuns();
[12806]756      SetEnabledStateOfControls();
[7980]757    }
[12841]758    private void problemComboBox_SelectedIndexChanged(object sender, EventArgs e) {
759      UpdateRuns();
760      SetEnabledStateOfControls();
761    }
[12631]762    private void dataTableComboBox_SelectedIndexChanged(object sender, EventArgs e) {
[12838]763      if (dataTableComboBox.SelectedIndex >= 0)
764        GenerateDefaultBudgets((string)dataTableComboBox.SelectedItem);
[14647]765      UpdateBestKnownQualities();
[12771]766      UpdateRuns();
[12806]767      SetEnabledStateOfControls();
[7980]768    }
[12771]769
770    private void logScalingCheckBox_CheckedChanged(object sender, EventArgs e) {
[14101]771      UpdateResultsByTarget();
[12838]772      byCostDataTable.VisualProperties.XAxisLogScale = byCostDataTable.Rows.Count > 0 && budgetLogScalingCheckBox.Checked;
[7980]773    }
[12804]774
[14101]775    private void boundShadingCheckBox_CheckedChanged(object sender, EventArgs e) {
776      UpdateResultsByTarget();
777    }
778
[12838]779    #region Event handlers for target analysis
[12804]780    private bool suppressTargetsEvents;
781    private void targetsTextBox_Validating(object sender, CancelEventArgs e) {
782      if (suppressTargetsEvents) return;
[12864]783      var targetStrings = targetsTextBox.Text.Split(new[] { '%', ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
784      var targetList = new List<decimal>();
[12804]785      foreach (var ts in targetStrings) {
[12864]786        decimal t;
787        if (!decimal.TryParse(ts, out t)) {
[12804]788          errorProvider.SetError(targetsTextBox, "Not all targets can be parsed: " + ts);
789          e.Cancel = true;
790          return;
791        }
[14647]792        if (targetsAreRelative)
793          targetList.Add(t / 100);
794        else targetList.Add(t);
[12804]795      }
796      if (targetList.Count == 0) {
797        errorProvider.SetError(targetsTextBox, "Give at least one target value!");
798        e.Cancel = true;
799        return;
800      }
801      e.Cancel = false;
802      errorProvider.SetError(targetsTextBox, null);
[14647]803      targets = targetsAreRelative ? targetList.Select(x => (double)x).OrderByDescending(x => x).ToArray() : targetList.Select(x => (double)x).ToArray();
[12888]804
[15125]805      SynchronizeTargetTextBox();
[12838]806      UpdateResultsByTarget();
[12806]807      SetEnabledStateOfControls();
[12804]808    }
[12806]809
[14101]810    private void aggregateTargetsCheckBox_CheckedChanged(object sender, EventArgs e) {
[12838]811      SuspendRepaint();
812      try {
813        UpdateResultsByTarget();
814      } finally { ResumeRepaint(true); }
815    }
816
[14652]817    private void relativeOrAbsoluteComboBox_SelectedIndexChanged(object sender, EventArgs e) {
[14647]818      if (suppressUpdates) return;
819      var pd = (ProblemInstance)problemComboBox.SelectedItem;
820      if (!double.IsNaN(pd.BestKnownQuality)) {
821        var max = pd.IsMaximization();
[14654]822        if (targetsAreRelative) targets = GetAbsoluteTargets(pd).ToArray();
[14647]823        else {
824          // Rounding to 5 digits since it's certainly appropriate for this application
825          if (pd.BestKnownQuality > 0) {
826            targets = targets.Select(x => Math.Round(max ? 1.0 - (x / pd.BestKnownQuality) : (x / pd.BestKnownQuality) - 1.0, 5)).ToArray();
827          } else if (pd.BestKnownQuality < 0) {
828            targets = targets.Select(x => Math.Round(!max ? 1.0 - (x / pd.BestKnownQuality) : (x / pd.BestKnownQuality) - 1.0, 5)).ToArray();
829          }
830        }
831      }
[14652]832      targetsAreRelative = (string)relativeOrAbsoluteComboBox.SelectedItem == "relative";
[15125]833      SynchronizeTargetTextBox();
834
[14647]835      try {
[15125]836        SuspendRepaint();
[14647]837        UpdateResultsByTarget();
[15125]838      } finally { ResumeRepaint(true); }
[14647]839    }
840
[12806]841    private void generateTargetsButton_Click(object sender, EventArgs e) {
[15125]842      if (targets == null) return;
843      decimal max = 10, min = 0, count = 10;
844      max = (decimal)targets.Max();
845      min = (decimal)targets.Min();
846      count = targets.Length - 1;
847      if (targetsAreRelative) {
848        max *= 100;
849        min *= 100;
[12806]850      }
851      using (var dialog = new DefineArithmeticProgressionDialog(false, min, max, (max - min) / count)) {
852        if (dialog.ShowDialog() == DialogResult.OK) {
853          if (dialog.Values.Any()) {
[15125]854            targets = targetsAreRelative
855              ? dialog.Values.OrderByDescending(x => x).Select(x => (double)x / 100.0).ToArray()
856              : dialog.Values.Select(x => (double)x).ToArray();
[12808]857
[15125]858            SynchronizeTargetTextBox();
[12838]859            UpdateResultsByTarget();
[12806]860            SetEnabledStateOfControls();
861          }
862        }
863      }
864    }
865
[12838]866    private void addTargetsAsResultButton_Click(object sender, EventArgs e) {
[12806]867      var table = (string)dataTableComboBox.SelectedItem;
[14647]868      if (string.IsNullOrEmpty(table)) return;
[14654]869     
[12806]870      foreach (var run in Content) {
[12808]871        if (!run.Results.ContainsKey(table)) continue;
[12806]872        var resultsTable = (IndexedDataTable<double>)run.Results[table];
873        var values = resultsTable.Rows.First().Values;
[14647]874        var pd = new ProblemInstance(run);
[15125]875        pd = problems.Single(x => x.Equals(pd));
[14647]876        var max = pd.IsMaximization();
[15125]877        var absTargets = GetAbsoluteTargetsWorstToBest(pd);
878
879        var prevIndex = 0;
880        for (var i = 0; i < absTargets.Length; i++) {
881          var absTarget = absTargets[i];
882          var index = values.BinarySearch(prevIndex, values.Count - prevIndex, Tuple.Create(0.0, absTarget), new TargetComparer(max));
883          if (index < 0) {
884            index = ~index;
885            if (index >= values.Count) break; // the target (and subsequent ones) wasn't achieved
[12806]886          }
[15125]887          var target = targetsAreRelative ? (targets[i] * 100) : absTarget;
888          run.Results[table + (targetsAreRelative ? ".RelTarget " : ".AbsTarget ") + target + (targetsAreRelative ? "%" : string.Empty)] = new DoubleValue(values[index].Item1);
889          prevIndex = index;
[12806]890        }
891      }
892    }
[14652]893
894    private void markerCheckBox_CheckedChanged(object sender, EventArgs e) {
895      SuspendRepaint();
896      try {
897        UpdateResultsByTarget();
898      } finally { ResumeRepaint(true); }
899    }
[15125]900
901    private void showLabelsCheckBox_CheckedChanged(object sender, EventArgs e) {
902      showLabelsInTargetChart = showLabelsCheckBox.Checked;
903      SuspendRepaint();
904      try {
905        UpdateResultsByTarget();
906      } finally { ResumeRepaint(true); }
907    }
[12838]908    #endregion
[12806]909
[12838]910    #region Event handlers for cost analysis
911    private bool suppressBudgetsEvents;
912    private void budgetsTextBox_Validating(object sender, CancelEventArgs e) {
913      if (suppressBudgetsEvents) return;
914      var budgetStrings = budgetsTextBox.Text.Split(new[] { ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
915      var budgetList = new List<double>();
916      foreach (var ts in budgetStrings) {
917        double b;
918        if (!double.TryParse(ts, out b)) {
[15125]919          errorProvider.SetError(budgetsTextBox, "Not all budgets can be parsed: " + ts);
[12838]920          e.Cancel = true;
921          return;
922        }
923        budgetList.Add(b);
924      }
925      if (budgetList.Count == 0) {
[15125]926        errorProvider.SetError(budgetsTextBox, "Give at least one budget value!");
[12838]927        e.Cancel = true;
928        return;
929      }
930      e.Cancel = false;
931      errorProvider.SetError(budgetsTextBox, null);
[15125]932      budgets = budgetList.OrderBy(x => x).ToArray();
933      try {
934        suppressBudgetsEvents = true;
935        budgetsTextBox.Text = string.Join(" ; ", budgets);
936      } finally { suppressBudgetsEvents = false; }
[12838]937      UpdateResultsByCost();
938      SetEnabledStateOfControls();
939    }
940
941    private void generateBudgetsButton_Click(object sender, EventArgs e) {
942      decimal max = 1, min = 0, count = 10;
943      if (budgets != null) {
944        max = (decimal)budgets.Max();
945        min = (decimal)budgets.Min();
946        count = budgets.Length;
947      } else if (Content.Count > 0 && dataTableComboBox.SelectedIndex >= 0) {
948        var table = (string)dataTableComboBox.SelectedItem;
949        min = (decimal)Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Min(y => y.Item1)).Min();
950        max = (decimal)Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Max(y => y.Item1)).Max();
[15125]951        count = 3;
[12838]952      }
953      using (var dialog = new DefineArithmeticProgressionDialog(false, min, max, (max - min) / count)) {
954        if (dialog.ShowDialog() == DialogResult.OK) {
955          if (dialog.Values.Any()) {
956            budgets = dialog.Values.OrderBy(x => x).Select(x => (double)x).ToArray();
957
[15125]958            try {
959              suppressBudgetsEvents = true;
960              budgetsTextBox.Text = string.Join(" ; ", budgets);
961            } finally { suppressBudgetsEvents = false; }
[12838]962
963            UpdateResultsByCost();
964            SetEnabledStateOfControls();
965          }
966        }
967      }
968    }
969
970    private void addBudgetsAsResultButton_Click(object sender, EventArgs e) {
[12806]971      var table = (string)dataTableComboBox.SelectedItem;
972
973      foreach (var run in Content) {
[12808]974        if (!run.Results.ContainsKey(table)) continue;
[12806]975        var resultsTable = (IndexedDataTable<double>)run.Results[table];
976        var values = resultsTable.Rows.First().Values;
[15125]977        var pd = new ProblemInstance(run);
978        pd = problems.Single(x => x.Equals(pd));
979
980        var prevIndex = 0;
981        foreach (var b in budgets) {
982          var index = values.BinarySearch(prevIndex, values.Count - prevIndex, Tuple.Create(b, 0.0), new CostComparer());
983          if (index < 0) {
984            index = ~index;
985            if (index >= values.Count) break; // the run wasn't long enough to use up budget b (or any subsequent larger one)
[12806]986          }
[15125]987          var v = values[index];
988          var tgt = targetsAreRelative ? CalculateRelativeDifference(pd.IsMaximization(), pd.BestKnownQuality, v.Item2) : v.Item2;
989          run.Results[table + (targetsAreRelative ? ".CostForRelTarget " : ".CostForAbsTarget ") + b] = new DoubleValue(tgt);
990          prevIndex = index;
[12806]991        }
992      }
993    }
[12838]994    #endregion
[12808]995
[12838]996    #region Helpers
[14647]997    private double CalculateRelativeDifference(bool maximization, double bestKnown, double fit) {
998      if (bestKnown == 0) {
999        // no relative difference with respect to bestKnown possible
1000        return maximization ? -fit : fit;
1001      }
1002      var absDiff = (fit - bestKnown);
1003      var relDiff = absDiff / bestKnown;
1004      if (maximization) {
1005        return bestKnown > 0 ? -relDiff : relDiff;
1006      } else {
1007        return bestKnown > 0 ? relDiff : -relDiff;
1008      }
1009    }
1010
1011    private Dictionary<ProblemInstance, double> CalculateBestTargetPerProblemInstance(string table) {
1012      if (table == null) table = string.Empty;
[12888]1013      return (from r in Content
1014              where r.Visible
[14647]1015              let pd = new ProblemInstance(r)
[12888]1016              let target = r.Parameters.ContainsKey("BestKnownQuality")
1017                           && r.Parameters["BestKnownQuality"] is DoubleValue
1018                ? ((DoubleValue)r.Parameters["BestKnownQuality"]).Value
[14647]1019                : (r.Results.ContainsKey(table) && r.Results[table] is IndexedDataTable<double>
1020                  ? ((IndexedDataTable<double>)r.Results[table]).Rows.First().Values.Last().Item2
1021                  : double.NaN)
[12888]1022              group target by pd into g
[14647]1023              select new { Problem = g.Key, Target = g.Any(x => !double.IsNaN(x)) ? (g.Key.IsMaximization() ? g.Max() : g.Where(x => !double.IsNaN(x)).Min()) : double.NaN })
[12888]1024        .ToDictionary(x => x.Problem, x => x.Target);
1025    }
1026
1027    private void UpdateRuns() {
1028      if (InvokeRequired) {
1029        Invoke((Action)UpdateRuns);
1030        return;
1031      }
1032      SuspendRepaint();
1033      try {
1034        UpdateResultsByTarget();
1035        UpdateResultsByCost();
1036      } finally { ResumeRepaint(true); }
1037    }
[14647]1038
1039    private void UpdateBestKnownQualities() {
1040      var table = (string)dataTableComboBox.SelectedItem;
1041      if (string.IsNullOrEmpty(table)) return;
1042
1043      var targetsPerProblem = CalculateBestTargetPerProblemInstance(table);
1044      foreach (var pd in problems) {
1045        double bkq;
1046        if (targetsPerProblem.TryGetValue(pd, out bkq))
1047          pd.BestKnownQuality = bkq;
1048        else pd.BestKnownQuality = double.NaN;
1049      }
1050    }
[12838]1051    #endregion
[12841]1052
[14101]1053    private void ConfigureSeries(Series series) {
[15125]1054      series.SmartLabelStyle.Enabled = showLabelsInTargetChart;
[14101]1055      series.SmartLabelStyle.AllowOutsidePlotArea = LabelOutsidePlotAreaStyle.No;
1056      series.SmartLabelStyle.CalloutLineAnchorCapStyle = LineAnchorCapStyle.None;
1057      series.SmartLabelStyle.CalloutLineColor = series.Color;
1058      series.SmartLabelStyle.CalloutLineWidth = 2;
1059      series.SmartLabelStyle.CalloutStyle = LabelCalloutStyle.Underlined;
1060      series.SmartLabelStyle.IsOverlappedHidden = false;
1061      series.SmartLabelStyle.MaxMovingDistance = 200;
1062      series.ToolTip = series.LegendText + " X = #VALX, Y = #VALY";
1063    }
[12841]1064
[14101]1065    private void chart_MouseDown(object sender, MouseEventArgs e) {
1066      HitTestResult result = targetChart.HitTest(e.X, e.Y);
1067      if (result.ChartElementType == ChartElementType.LegendItem) {
1068        ToggleTargetChartSeriesVisible(result.Series);
1069      }
1070    }
1071    private void chart_MouseMove(object sender, MouseEventArgs e) {
1072      HitTestResult result = targetChart.HitTest(e.X, e.Y);
1073      if (result.ChartElementType == ChartElementType.LegendItem)
1074        this.Cursor = Cursors.Hand;
1075      else
1076        this.Cursor = Cursors.Default;
1077    }
1078    private void chart_CustomizeLegend(object sender, CustomizeLegendEventArgs e) {
1079      foreach (LegendItem legendItem in e.LegendItems) {
1080        var series = targetChart.Series[legendItem.SeriesName];
1081        if (series != null) {
1082          bool seriesIsInvisible = invisibleTargetSeries.Any(x => x.Name == series.Name);
1083          foreach (LegendCell cell in legendItem.Cells) {
1084            cell.ForeColor = seriesIsInvisible ? Color.Gray : Color.Black;
1085          }
1086        }
1087      }
1088    }
1089
1090    private void ToggleTargetChartSeriesVisible(Series series) {
1091      var indexList = invisibleTargetSeries.FindIndex(x => x.Name == series.Name);
1092      var indexChart = targetChart.Series.IndexOf(series);
1093      if (targetChart.Series.Count == 1) targetChart.ChartAreas[0].AxisX.IsLogarithmic = false;
1094      targetChart.Series.RemoveAt(indexChart);
1095      var s = indexList >= 0 ? invisibleTargetSeries[indexList] : new Series(series.Name) {
1096        Color = series.Color,
1097        ChartType = series.ChartType,
1098        BorderWidth = series.BorderWidth,
1099        BorderDashStyle = series.BorderDashStyle
1100      };
1101      if (indexList < 0) {
1102        // hide
1103        invisibleTargetSeries.Add(series);
1104        var shadeSeries = targetChart.Series.FirstOrDefault(x => x.Name == series.Name + "-range");
1105        if (shadeSeries != null) {
1106          if (targetChart.Series.Count == 1) targetChart.ChartAreas[0].AxisX.IsLogarithmic = false;
1107          targetChart.Series.Remove(shadeSeries);
1108          invisibleTargetSeries.Add(shadeSeries);
1109          indexChart--;
1110        }
1111      } else {
1112        // show
1113        invisibleTargetSeries.RemoveAt(indexList);
1114        var shadeSeries = invisibleTargetSeries.FirstOrDefault(x => x.Name == series.Name + "-range");
1115        if (shadeSeries != null) {
1116          invisibleTargetSeries.Remove(shadeSeries);
1117          InsertOrAddSeries(indexChart, shadeSeries);
1118          indexChart++;
1119        }
1120      }
1121      InsertOrAddSeries(indexChart, s);
1122      targetChart.ChartAreas[0].AxisX.IsLogarithmic = CanDisplayLogarithmic();
1123    }
1124
1125    private bool CanDisplayLogarithmic() {
1126      return targetLogScalingCheckBox.Checked
1127        && targetChart.Series.Count > 0 // must have a series
1128        && targetChart.Series.Any(x => x.Points.Count > 0) // at least one series must have points
1129        && targetChart.Series.All(s => s.Points.All(p => p.XValue > 0)); // all points must be positive
1130    }
1131
1132    private void InsertOrAddSeries(int index, Series s) {
1133      if (targetChart.Series.Count <= index)
1134        targetChart.Series.Add(s);
1135      else targetChart.Series.Insert(index, s);
1136    }
1137
[14775]1138    private class AlgorithmInstance : INotifyPropertyChanged {
1139      private string name;
1140      public string Name {
1141        get { return name; }
1142        set {
1143          if (name == value) return;
1144          name = value;
1145          OnPropertyChanged("Name");
1146        }
1147      }
1148
1149      private Dictionary<ProblemInstance, List<IRun>> performanceData;
1150
1151      public int GetNumberOfProblemInstances() {
1152        return performanceData.Count;
1153      }
1154
1155      public IEnumerable<ProblemInstance> GetProblemInstances() {
1156        return performanceData.Keys;
1157      }
1158
1159      public int GetNumberOfRuns(ProblemInstance p) {
1160        if (p == ProblemInstance.MatchAll) return performanceData.Select(x => x.Value.Count).Sum();
1161        List<IRun> runs;
1162        if (performanceData.TryGetValue(p, out runs))
1163          return runs.Count;
1164
1165        return 0;
1166      }
1167
1168      public IEnumerable<IRun> GetRuns(ProblemInstance p) {
1169        if (p == ProblemInstance.MatchAll) return performanceData.SelectMany(x => x.Value);
1170        List<IRun> runs;
1171        if (performanceData.TryGetValue(p, out runs))
1172          return runs;
1173
1174        return Enumerable.Empty<IRun>();
1175      }
1176
1177      public AlgorithmInstance(string name, IEnumerable<IRun> runs, IEnumerable<ProblemInstance> problems) {
1178        this.name = name;
1179
1180        var pDict = problems.ToDictionary(r => r, _ => new List<IRun>());
1181        foreach (var y in runs) {
1182          var pd = new ProblemInstance(y);
1183          List<IRun> l;
1184          if (pDict.TryGetValue(pd, out l)) l.Add(y);
1185        }
1186        performanceData = pDict.Where(x => x.Value.Count > 0).ToDictionary(x => x.Key, x => x.Value);
1187      }
1188
1189      public override bool Equals(object obj) {
1190        var other = obj as AlgorithmInstance;
1191        if (other == null) return false;
1192        return name == other.name;
1193      }
1194
1195      public override int GetHashCode() {
1196        return name.GetHashCode();
1197      }
1198
1199      public event PropertyChangedEventHandler PropertyChanged;
1200      protected virtual void OnPropertyChanged(string propertyName = null) {
1201        var handler = PropertyChanged;
1202        if (handler != null) handler(this, new PropertyChangedEventArgs(propertyName));
1203      }
1204    }
1205
[14647]1206    private class ProblemInstance : INotifyPropertyChanged {
[12841]1207      private readonly bool matchAll;
[14647]1208      public static readonly ProblemInstance MatchAll = new ProblemInstance() {
[12864]1209        ProblemName = "All with Best-Known"
1210      };
[12841]1211
[14647]1212      private ProblemInstance() {
[12841]1213        ProblemType = string.Empty;
1214        ProblemName = string.Empty;
1215        Evaluator = string.Empty;
[13583]1216        Maximization = string.Empty;
[12841]1217        DisplayProblemType = false;
1218        DisplayProblemName = false;
1219        DisplayEvaluator = false;
[13583]1220        DisplayMaximization = false;
[12841]1221        matchAll = true;
[14647]1222        BestKnownQuality = double.NaN;
[12841]1223      }
1224
[14647]1225      public ProblemInstance(IRun run) {
[12841]1226        ProblemType = GetStringValueOrEmpty(run, "Problem Type");
1227        ProblemName = GetStringValueOrEmpty(run, "Problem Name");
1228        Evaluator = GetStringValueOrEmpty(run, "Evaluator");
[13583]1229        Maximization = GetMaximizationValueOrEmpty(run, "Maximization");
[12841]1230        DisplayProblemType = !string.IsNullOrEmpty(ProblemType);
1231        DisplayProblemName = !string.IsNullOrEmpty(ProblemName);
1232        DisplayEvaluator = !string.IsNullOrEmpty(Evaluator);
[13583]1233        DisplayMaximization = !string.IsNullOrEmpty(Maximization);
[12841]1234        matchAll = false;
[14647]1235        BestKnownQuality = GetDoubleValueOrNaN(run, "BestKnownQuality");
[12841]1236      }
1237
[14647]1238      private bool displayProblemType;
1239      public bool DisplayProblemType {
1240        get { return displayProblemType; }
1241        set {
1242          if (displayProblemType == value) return;
1243          displayProblemType = value;
1244          OnPropertyChanged("DisplayProblemType");
1245        }
1246      }
1247      private string problemType;
1248      public string ProblemType {
1249        get { return problemType; }
1250        set {
1251          if (problemType == value) return;
1252          problemType = value;
1253          OnPropertyChanged("ProblemType");
1254        }
1255      }
1256      private bool displayProblemName;
1257      public bool DisplayProblemName {
1258        get { return displayProblemName; }
1259        set {
1260          if (displayProblemName == value) return;
1261          displayProblemName = value;
1262          OnPropertyChanged("DisplayProblemName");
1263        }
1264      }
1265      private string problemName;
1266      public string ProblemName {
1267        get { return problemName; }
1268        set {
1269          if (problemName == value) return;
1270          problemName = value;
1271          OnPropertyChanged("ProblemName");
1272        }
1273      }
1274      private bool displayEvaluator;
1275      public bool DisplayEvaluator {
1276        get { return displayEvaluator; }
1277        set {
1278          if (displayEvaluator == value) return;
1279          displayEvaluator = value;
1280          OnPropertyChanged("DisplayEvaluator");
1281        }
1282      }
1283      private string evaluator;
1284      public string Evaluator {
1285        get { return evaluator; }
1286        set {
1287          if (evaluator == value) return;
1288          evaluator = value;
1289          OnPropertyChanged("Evaluator");
1290        }
1291      }
1292      private bool displayMaximization;
1293      public bool DisplayMaximization {
1294        get { return displayMaximization; }
1295        set {
1296          if (displayMaximization == value) return;
1297          displayMaximization = value;
1298          OnPropertyChanged("DisplayMaximization");
1299        }
1300      }
1301      private string maximization;
1302      public string Maximization {
1303        get { return maximization; }
1304        set {
1305          if (maximization == value) return;
1306          maximization = value;
1307          OnPropertyChanged("Maximization");
1308        }
1309      }
1310      private double bestKnownQuality;
1311      public double BestKnownQuality {
1312        get { return bestKnownQuality; }
1313        set {
1314          if (bestKnownQuality == value) return;
1315          bestKnownQuality = value;
1316          OnPropertyChanged("BestKnownQuality");
1317        }
1318      }
[12841]1319
[13583]1320      public bool IsMaximization() {
1321        return Maximization == "MAX";
1322      }
1323
[12841]1324      public bool Match(IRun run) {
1325        return matchAll ||
1326               GetStringValueOrEmpty(run, "Problem Type") == ProblemType
1327               && GetStringValueOrEmpty(run, "Problem Name") == ProblemName
[13583]1328               && GetStringValueOrEmpty(run, "Evaluator") == Evaluator
1329               && GetMaximizationValueOrEmpty(run, "Maximization") == Maximization;
[12841]1330      }
1331
[14647]1332      private double GetDoubleValueOrNaN(IRun run, string key) {
1333        IItem param;
1334        if (run.Parameters.TryGetValue(key, out param)) {
1335          var dv = param as DoubleValue;
1336          return dv != null ? dv.Value : double.NaN;
1337        }
1338        return double.NaN;
1339      }
1340
[12841]1341      private string GetStringValueOrEmpty(IRun run, string key) {
[14647]1342        IItem param;
1343        if (run.Parameters.TryGetValue(key, out param)) {
1344          var sv = param as StringValue;
1345          return sv != null ? sv.Value : string.Empty;
1346        }
1347        return string.Empty;
[12841]1348      }
1349
[13583]1350      private string GetMaximizationValueOrEmpty(IRun run, string key) {
[14647]1351        IItem param;
1352        if (run.Parameters.TryGetValue(key, out param)) {
1353          var bv = param as BoolValue;
1354          return bv != null ? (bv.Value ? "MAX" : "MIN") : string.Empty;
1355        }
1356        return string.Empty;
[13583]1357      }
1358
[12841]1359      public override bool Equals(object obj) {
[14647]1360        var other = obj as ProblemInstance;
[12841]1361        if (other == null) return false;
1362        return ProblemType == other.ProblemType
1363               && ProblemName == other.ProblemName
[13583]1364               && Evaluator == other.Evaluator
1365               && Maximization == other.Maximization;
[12841]1366      }
1367
1368      public override int GetHashCode() {
[13583]1369        return ProblemType.GetHashCode() ^ ProblemName.GetHashCode() ^ Evaluator.GetHashCode() ^ Maximization.GetHashCode();
[12841]1370      }
1371
1372      public override string ToString() {
1373        return string.Join("  --  ", new[] {
1374          (DisplayProblemType ? ProblemType : string.Empty),
1375          (DisplayProblemName ? ProblemName : string.Empty),
[13583]1376          (DisplayEvaluator ? Evaluator : string.Empty),
[14647]1377          (DisplayMaximization ? Maximization : string.Empty),
1378          !double.IsNaN(BestKnownQuality) ? BestKnownQuality.ToString(CultureInfo.CurrentCulture.NumberFormat) : string.Empty }.Where(x => !string.IsNullOrEmpty(x)));
[12841]1379      }
[14647]1380
1381      public event PropertyChangedEventHandler PropertyChanged;
1382      protected virtual void OnPropertyChanged(string propertyName = null) {
1383        var handler = PropertyChanged;
1384        if (handler != null) handler(this, new PropertyChangedEventArgs(propertyName));
1385      }
[12841]1386    }
[15125]1387
1388    private class CostComparer : Comparer<Tuple<double, double>> {
1389      public override int Compare(Tuple<double, double> x, Tuple<double, double> y) {
1390        return x.Item1.CompareTo(y.Item1);
1391      }
1392    }
1393
1394    private class TargetComparer : Comparer<Tuple<double, double>> {
[15126]1395      public bool Maximization { get; private set; }
[15125]1396      public TargetComparer(bool maximization) {
1397        Maximization = maximization;
1398      }
1399
1400      public override int Compare(Tuple<double, double> x, Tuple<double, double> y) {
1401        return Maximization ? x.Item2.CompareTo(y.Item2) : y.Item2.CompareTo(x.Item2);
1402      }
1403    }
[7980]1404  }
1405}
Note: See TracBrowser for help on using the repository browser.