Free cookie consent management tool by TermsFeed Policy Generator

Changeset 16716


Ignore:
Timestamp:
03/27/19 17:13:54 (6 years ago)
Author:
abeham
Message:

#3000:

  • Added a lof changes from #1614 branch not yet merged to trunk
  • Fixed problem with missed targets at simultaneous points to hits
Location:
trunk/HeuristicLab.Optimization.Views/3.3/RunCollectionViews
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • trunk/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.Designer.cs

    r16565 r16716  
    11#region License Information
    22/* HeuristicLab
    3  * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
     3 * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
    44 *
    55 * This file is part of HeuristicLab.
     
    8787      // dataTableComboBox
    8888      //
    89       this.dataTableComboBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left)
     89      this.dataTableComboBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left) 
    9090            | System.Windows.Forms.AnchorStyles.Right)));
    9191      this.dataTableComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
     
    117117      // groupComboBox
    118118      //
    119       this.groupComboBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left)
     119      this.groupComboBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left) 
    120120            | System.Windows.Forms.AnchorStyles.Right)));
    121121      this.groupComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
     
    141141      // targetsTextBox
    142142      //
    143       this.targetsTextBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left)
     143      this.targetsTextBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left) 
    144144            | System.Windows.Forms.AnchorStyles.Right)));
    145145      this.targetsTextBox.Location = new System.Drawing.Point(59, 8);
     
    198198      // budgetsTextBox
    199199      //
    200       this.budgetsTextBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left)
     200      this.budgetsTextBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left) 
    201201            | System.Windows.Forms.AnchorStyles.Right)));
    202202      this.budgetsTextBox.Location = new System.Drawing.Point(59, 8);
     
    233233      // tabControl
    234234      //
    235       this.tabControl.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom)
    236             | System.Windows.Forms.AnchorStyles.Left)
     235      this.tabControl.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) 
     236            | System.Windows.Forms.AnchorStyles.Left) 
    237237            | System.Windows.Forms.AnchorStyles.Right)));
    238238      this.tabControl.Controls.Add(this.byTargetTabPage);
     
    247247      // byTargetTabPage
    248248      //
    249       this.byTargetTabPage.BackColor = System.Drawing.SystemColors.Window;
    250249      this.byTargetTabPage.Controls.Add(this.relativeOrAbsoluteComboBox);
    251250      this.byTargetTabPage.Controls.Add(this.targetChart);
     
    265264      this.byTargetTabPage.TabIndex = 0;
    266265      this.byTargetTabPage.Text = "Performance by Target";
     266      this.byTargetTabPage.UseVisualStyleBackColor = true;
    267267      //
    268268      // relativeOrAbsoluteComboBox
     
    282282      // targetChart
    283283      //
    284       this.targetChart.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom)
    285             | System.Windows.Forms.AnchorStyles.Left)
     284      this.targetChart.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) 
     285            | System.Windows.Forms.AnchorStyles.Left) 
    286286            | System.Windows.Forms.AnchorStyles.Right)));
    287287      chartArea2.AxisX.IsStartedFromZero = false;
     
    352352      // byCostTabPage
    353353      //
    354       this.byCostTabPage.BackColor = System.Drawing.SystemColors.Window;
    355354      this.byCostTabPage.Controls.Add(this.byCostViewHost);
    356355      this.byCostTabPage.Controls.Add(this.budgetLogScalingCheckBox);
     
    365364      this.byCostTabPage.TabIndex = 1;
    366365      this.byCostTabPage.Text = "Performance by Cost";
     366      this.byCostTabPage.UseVisualStyleBackColor = true;
    367367      //
    368368      // byCostViewHost
    369369      //
    370       this.byCostViewHost.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom)
    371             | System.Windows.Forms.AnchorStyles.Left)
     370      this.byCostViewHost.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) 
     371            | System.Windows.Forms.AnchorStyles.Left) 
    372372            | System.Windows.Forms.AnchorStyles.Right)));
    373373      this.byCostViewHost.Caption = "View";
     
    407407      // byTableTabPage
    408408      //
    409       this.byTableTabPage.BackColor = System.Drawing.SystemColors.Window;
    410409      this.byTableTabPage.Controls.Add(this.ertTableView);
    411410      this.byTableTabPage.Location = new System.Drawing.Point(4, 22);
     
    415414      this.byTableTabPage.TabIndex = 2;
    416415      this.byTableTabPage.Text = "Expected Runtime Tables";
     416      this.byTableTabPage.UseVisualStyleBackColor = true;
    417417      //
    418418      // ertTableView
     
    431431      // problemComboBox
    432432      //
    433       this.problemComboBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left)
     433      this.problemComboBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left) 
    434434            | System.Windows.Forms.AnchorStyles.Right)));
    435435      this.problemComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
  • trunk/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs

    r16565 r16716  
    11#region License Information
    22/* HeuristicLab
    3  * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
     3 * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
    44 *
    55 * This file is part of HeuristicLab.
     
    2121
    2222using System;
     23using System.Collections;
    2324using System.Collections.Generic;
    2425using System.ComponentModel;
     
    3031using HeuristicLab.Analysis;
    3132using HeuristicLab.Collections;
     33using HeuristicLab.Common;
    3234using HeuristicLab.Core;
    3335using HeuristicLab.Core.Views;
     
    7880    }
    7981
     82    private List<AlgorithmInstance> groups;
     83
    8084    private double[] targets;
    8185    private double[] budgets;
     
    8488    private readonly BindingList<ProblemInstance> problems;
    8589
    86     private bool suppressUpdates;
     90    private bool updateInProgress;
     91    private bool suppressContentEvents;
    8792    private readonly IndexedDataTable<double> byCostDataTable;
    8893    public IndexedDataTable<double> ByCostDataTable {
     
    9499      invisibleTargetSeries = new List<Series>();
    95100
    96       targetChart.CustomizeAllChartAreas();
    97       targetChart.ChartAreas[0].CursorX.Interval = 1;
    98       targetChart.SuppressExceptions = true;
    99       byCostDataTable = new IndexedDataTable<double>("ECDF by Cost", "A data table containing the ECDF of function values (relative to best-known).") {
    100         VisualProperties = {
    101           YAxisTitle = "Proportion of runs",
    102           YAxisMinimumFixedValue = 0,
    103           YAxisMinimumAuto = false,
    104           YAxisMaximumFixedValue = 1,
    105           YAxisMaximumAuto = false
    106         }
    107       };
    108       byCostViewHost.Content = byCostDataTable;
    109       suppressUpdates = true;
    110       relativeOrAbsoluteComboBox.SelectedItem = targetsAreRelative ? "relative" : "absolute";
    111       problems = new BindingList<ProblemInstance>();
    112       problemComboBox.DataSource = new BindingSource() { DataSource = problems };
    113       problemComboBox.DataBindings.DefaultDataSourceUpdateMode = DataSourceUpdateMode.OnPropertyChanged;
    114       suppressUpdates = false;
     101      try {
     102        updateInProgress = true;
     103        targetChart.CustomizeAllChartAreas();
     104        targetChart.ChartAreas[0].CursorX.Interval = 1;
     105        targetChart.SuppressExceptions = true;
     106        byCostDataTable = new IndexedDataTable<double>("ECDF by Cost", "A data table containing the ECDF of function values (relative to best-known).") {
     107          VisualProperties = {
     108            YAxisTitle = "Proportion of runs",
     109            YAxisMinimumFixedValue = 0,
     110            YAxisMinimumAuto = false,
     111            YAxisMaximumFixedValue = 1,
     112            YAxisMaximumAuto = false
     113          }
     114        };
     115        byCostViewHost.Content = byCostDataTable;
     116
     117        relativeOrAbsoluteComboBox.SelectedItem = targetsAreRelative ? "relative" : "absolute";
     118        problems = new BindingList<ProblemInstance>();
     119        problemComboBox.DataSource = new BindingSource() { DataSource = problems };
     120        problemComboBox.DataBindings.DefaultDataSourceUpdateMode = DataSourceUpdateMode.OnPropertyChanged;
     121      } finally { updateInProgress = false; }
    115122    }
    116123
     
    134141
    135142    private void Content_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IRun> e) {
    136       if (suppressUpdates) return;
     143      foreach (var run in e.Items) RegisterRunEvents(run);
     144      if (suppressContentEvents) return;
    137145      if (InvokeRequired) {
    138146        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_ItemsAdded), sender, e);
    139147        return;
    140148      }
    141       UpdateGroupAndProblemComboBox();
    142       UpdateDataTableComboBox();
    143       foreach (var run in e.Items)
    144         RegisterRunEvents(run);
     149      if (updateInProgress) return;
     150      try {
     151        updateInProgress = true;
     152        UpdateComboBoxes();
     153        GroupRuns();
     154      } finally { updateInProgress = false; }
    145155    }
    146156    private void Content_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IRun> e) {
    147       if (suppressUpdates) return;
     157      foreach (var run in e.Items) DeregisterRunEvents(run);
     158      if (suppressContentEvents) return;
    148159      if (InvokeRequired) {
    149160        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_ItemsRemoved), sender, e);
    150161        return;
    151162      }
    152       UpdateGroupAndProblemComboBox();
    153       UpdateDataTableComboBox();
    154       foreach (var run in e.Items)
    155         DeregisterRunEvents(run);
     163      if (updateInProgress) return;
     164      try {
     165        updateInProgress = true;
     166        UpdateComboBoxes();
     167        GroupRuns();
     168      } finally { updateInProgress = false; }
    156169    }
    157170    private void Content_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
    158       if (suppressUpdates) return;
     171      foreach (var run in e.OldItems) DeregisterRunEvents(run);
     172      foreach (var run in e.Items) RegisterRunEvents(run);
     173      if (suppressContentEvents) return;
    159174      if (InvokeRequired) {
    160175        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset), sender, e);
    161176        return;
    162177      }
    163       UpdateGroupAndProblemComboBox();
    164       UpdateDataTableComboBox();
    165       foreach (var run in e.OldItems)
    166         DeregisterRunEvents(run);
     178      if (updateInProgress) return;
     179      try {
     180        updateInProgress = true;
     181        UpdateComboBoxes();
     182        GroupRuns();
     183      } finally { updateInProgress = false; }
    167184    }
    168185    private void Content_AlgorithmNameChanged(object sender, EventArgs e) {
     
    176193        return;
    177194      }
    178       suppressUpdates = Content.UpdateOfRunsInProgress;
    179       if (!suppressUpdates) {
    180         UpdateDataTableComboBox();
    181         UpdateGroupAndProblemComboBox();
    182         UpdateRuns();
     195      suppressContentEvents = Content.UpdateOfRunsInProgress;
     196      if (!suppressContentEvents) {
     197        if (updateInProgress) return;
     198        try {
     199          updateInProgress = true;
     200          UpdateComboBoxes();
     201          GroupRuns();
     202        } finally { updateInProgress = false; }
    183203      }
    184204    }
     
    191211    }
    192212    private void run_PropertyChanged(object sender, PropertyChangedEventArgs e) {
    193       if (suppressUpdates) return;
     213      if (suppressContentEvents) return;
    194214      if (InvokeRequired) {
    195215        Invoke((Action<object, PropertyChangedEventArgs>)run_PropertyChanged, sender, e);
    196216      } else {
    197         if (e.PropertyName == "Visible")
    198           UpdateRuns();
     217        if (e.PropertyName == "Visible") {
     218          if (updateInProgress) return;
     219          try {
     220            updateInProgress = true;
     221            UpdateRuns();
     222          } finally { updateInProgress = false; }
     223        }
    199224      }
    200225    }
     
    213238      UpdateCaption();
    214239      if (Content != null) {
    215         UpdateGroupAndProblemComboBox();
    216         UpdateDataTableComboBox();
    217       }
    218     }
    219 
    220 
    221     private void UpdateGroupAndProblemComboBox() {
     240        try {
     241          updateInProgress = true;
     242          UpdateComboBoxes();
     243          UpdateRuns();
     244        } finally { updateInProgress = false; }
     245      }
     246    }
     247
     248
     249    private void UpdateComboBoxes() {
    222250      var selectedGroupItem = (string)groupComboBox.SelectedItem;
    223251
     
    232260      }
    233261
    234       var table = (string)dataTableComboBox.SelectedItem;
    235       var problemDict = CalculateBestTargetPerProblemInstance(table);
    236 
    237       var problemTypesDifferent = problemDict.Keys.Select(x => x.ProblemType).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
    238       var problemNamesDifferent = problemDict.Keys.Select(x => x.ProblemName).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
    239       var evaluatorDifferent = problemDict.Keys.Select(x => x.Evaluator).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
    240       var maximizationDifferent = problemDict.Keys.Select(x => x.Maximization).Distinct().Count() > 1;
    241       var allEqual = !problemTypesDifferent && !problemNamesDifferent && !evaluatorDifferent && !maximizationDifferent;
    242 
    243       var selectedProblemItem = (ProblemInstance)problemComboBox.SelectedItem;
    244       problemComboBox.DataSource = null;
    245       problemComboBox.Items.Clear();
    246       problems.Clear();
    247       problems.Add(ProblemInstance.MatchAll);
    248       problemComboBox.DataSource = new BindingSource() { DataSource = problems };
    249       problemComboBox.DataBindings.DefaultDataSourceUpdateMode = DataSourceUpdateMode.OnPropertyChanged;
    250       if (problems[0].Equals(selectedProblemItem)) problemComboBox.SelectedItem = problems[0];
    251       foreach (var p in problemDict.ToList()) {
    252         p.Key.BestKnownQuality = p.Value;
    253         p.Key.DisplayProblemType = problemTypesDifferent;
    254         p.Key.DisplayProblemName = problemNamesDifferent || allEqual;
    255         p.Key.DisplayEvaluator = evaluatorDifferent;
    256         p.Key.DisplayMaximization = maximizationDifferent;
    257         problems.Add(p.Key);
    258         if (p.Key.Equals(selectedProblemItem)) problemComboBox.SelectedItem = p.Key;
    259       }
    260       SetEnabledStateOfControls();
    261     }
    262 
    263     private void UpdateDataTableComboBox() {
    264       string selectedItem = (string)dataTableComboBox.SelectedItem;
     262      string selectedDataTable = (string)dataTableComboBox.SelectedItem;
    265263
    266264      dataTableComboBox.Items.Clear();
     
    271269
    272270      dataTableComboBox.Items.AddRange(dataTables);
    273       if (selectedItem != null && dataTableComboBox.Items.Contains(selectedItem)) {
    274         dataTableComboBox.SelectedItem = selectedItem;
     271      if (selectedDataTable != null && dataTableComboBox.Items.Contains(selectedDataTable)) {
     272        dataTableComboBox.SelectedItem = selectedDataTable;
    275273      } else if (dataTableComboBox.Items.Count > 0) {
    276274        dataTableComboBox.SelectedItem = dataTableComboBox.Items[0];
    277275      }
     276
     277      var selectedProblemItem = (ProblemInstance)problemComboBox.SelectedItem;
     278     
     279      UpdateProblemInstances();
     280     
     281      foreach (var p in problems) {
     282        if (p.Equals(selectedProblemItem))
     283          problemComboBox.SelectedItem = p;
     284      }
     285
     286      if (selectedProblemItem == null && problems.Count > 1) problemComboBox.SelectedItem = problems[1];
     287
     288      SetEnabledStateOfControls();
    278289    }
    279290
     
    288299    }
    289300
    290     private IEnumerable<AlgorithmInstance> GroupRuns() {
     301    private void GroupRuns() {
     302      groups = new List<AlgorithmInstance>();
     303
    291304      var table = (string)dataTableComboBox.SelectedItem;
    292       if (string.IsNullOrEmpty(table)) yield break;
     305      if (string.IsNullOrEmpty(table)) return;
    293306
    294307      var selectedGroup = (string)groupComboBox.SelectedItem;
    295       if (string.IsNullOrEmpty(selectedGroup)) yield break;
     308      if (string.IsNullOrEmpty(selectedGroup)) return;
    296309
    297310      var selectedProblem = (ProblemInstance)problemComboBox.SelectedItem;
    298       if (selectedProblem == null) yield break;
    299      
    300       foreach (var x in (from r in Content
    301                          where (selectedGroup == AllInstances || r.Parameters.ContainsKey(selectedGroup))
    302                            && selectedProblem.Match(r)
    303                            && r.Results.ContainsKey(table)
    304                            && r.Visible
    305                          let key = selectedGroup == AllInstances ? AllInstances : r.Parameters[selectedGroup].ToString()
    306                          group r by key into g
    307                          select new AlgorithmInstance(g.Key, g, problems))) {
    308         yield return x;
    309       }
     311      if (selectedProblem == null) return;
     312
     313      foreach (var alg in from r in Content
     314                          where (selectedGroup == AllInstances || r.Parameters.ContainsKey(selectedGroup))
     315                            && r.Visible
     316                            && selectedProblem.Match(r)
     317                          let key = selectedGroup == AllInstances
     318                            ? AllInstances : r.Parameters[selectedGroup].ToString()
     319                          group r by key into g
     320                          select g) {
     321        var trials = (from run in alg
     322                     from pd in problems.Skip(1) // exclude artificial match all
     323                     where pd.Match(run) && run.Results.ContainsKey(table)
     324                     let cgraph = run.Results[table] as IndexedDataTable<double>
     325                     where cgraph != null && cgraph.Rows.Count > 0
     326                        && cgraph.Rows.First().Values.Count > 0
     327                     group cgraph by pd into g
     328                     select g).ToList();
     329
     330        if (trials.Count == 0) continue;
     331        groups.Add(new AlgorithmInstance(alg.Key, trials));
     332      }
     333    }
     334
     335    private void UpdateProblemInstances() {
     336      try {
     337        problems.Clear();
     338        var table = (string)dataTableComboBox.SelectedItem;
     339       
     340        var problemDict = CalculateBestTargetPerProblemInstance(table);
     341
     342        var problemTypesDifferent = problemDict.Keys.Select(x => x.ProblemType).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
     343        var problemNamesDifferent = problemDict.Keys.Select(x => x.ProblemName).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
     344        var evaluatorDifferent = problemDict.Keys.Select(x => x.Evaluator).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
     345        var maximizationDifferent = problemDict.Keys.Select(x => x.Maximization).Distinct().Count() > 1;
     346        var allEqual = !problemTypesDifferent && !problemNamesDifferent && !evaluatorDifferent && !maximizationDifferent;
     347
     348        problems.Add(ProblemInstance.MatchAll);
     349        foreach (var p in problemDict.OrderBy(x => x.Key.ProblemName, new NaturalStringComparer()).ToList()) {
     350          p.Key.BestKnownQuality = p.Value;
     351          p.Key.DisplayProblemType = problemTypesDifferent;
     352          p.Key.DisplayProblemName = problemNamesDifferent || allEqual;
     353          p.Key.DisplayEvaluator = evaluatorDifferent;
     354          p.Key.DisplayMaximization = maximizationDifferent;
     355          problems.Add(p.Key);
     356        }
     357      } finally { ((BindingSource)problemComboBox.DataSource).ResetBindings(false); }
    310358    }
    311359
     
    321369
    322370      if (targets == null) GenerateDefaultTargets();
    323 
    324       var algInstances = GroupRuns().ToList();
    325       if (algInstances.Count == 0) return;
     371     
     372      if (groups.Count == 0) return;
    326373
    327374      var xAxisTitles = new HashSet<string>();
     
    337384      var aggregate = aggregateTargetsCheckBox.Checked;
    338385      double minEff = double.MaxValue, maxEff = double.MinValue;
    339       foreach (var alg in algInstances) {
     386      foreach (var alg in groups) {
    340387        var noRuns = 0;
    341388        SortedList<double, int> epdfHits = null, epdfMisses = null;
     
    344391          misses[alg.Name] = epdfMisses = new SortedList<double, int>();
    345392        }
    346         foreach (var problem in alg.GetProblemInstances()) {
    347           var max = problem.IsMaximization();
     393        foreach (var problem in alg.GetCases().Where(x => x.Maximization.HasValue)) {
     394          var max = problem.Maximization.Value;
    348395          var absTargets = GetAbsoluteTargets(problem).ToArray();
    349           foreach (var run in alg.GetRuns(problem)) {
     396          foreach (var run in alg.GetTrials(problem)) {
    350397            noRuns++;
    351             var resultsTable = (IndexedDataTable<double>)run.Results[table];
    352             xAxisTitles.Add(resultsTable.VisualProperties.XAxisTitle);
    353 
    354             var efforts = absTargets.Select(t => GetEffortToHitTarget(resultsTable.Rows.First().Values, t, max)).ToArray();
     398            xAxisTitles.Add(run.XAxisName);
     399
     400            var efforts = absTargets.Select(t => GetEffortToHitTarget(run, t, max)).ToArray();
    355401            minEff = Math.Min(minEff, efforts.Min(x => x.Item2));
    356402            maxEff = Math.Max(maxEff, efforts.Max(x => x.Item2));
     
    391437      targetChart.ChartAreas[0].CursorY.Interval = 0.05;
    392438
    393       UpdateErtTables(algInstances);
     439      UpdateErtTables();
    394440    }
    395441
     
    401447      foreach (var list in hits) {
    402448        var row = new Series(list.Key) {
    403           ChartType = SeriesChartType.StepLine,
     449          ChartType = list.Value.Count > 1000 ? SeriesChartType.FastLine : SeriesChartType.StepLine,
    404450          BorderWidth = 3,
    405451          Color = colors[colorCount],
     
    423469        foreach (var h in list.Value) {
    424470          var prevmissedecdf = missedecdf;
    425           while (moreMisses && iter.Current.Key <= h.Key) {
     471          while (moreMisses && iter.Current.Key < h.Key) {
    426472            if (!labelPrinted && row.Points.Count > 0) {
    427473              var point = row.Points.Last();
     
    438484            if (row.Points.Count > 0 && row.Points.Last().XValue == iter.Current.Key) {
    439485              row.Points.Last().SetValueY(ecdf / movingTargets);
    440               row.Points.Last().Color = Color.FromArgb(255 - (int)Math.Floor(255 * (prevmissedecdf / totalTargets)), colors[colorCount]);
    441486            } else {
    442               var dp = new DataPoint(iter.Current.Key, ecdf / movingTargets) {
    443                 Color = Color.FromArgb(255 - (int)Math.Floor(255 * (prevmissedecdf / totalTargets)), colors[colorCount])
    444               };
     487              var dp = new DataPoint(iter.Current.Key, ecdf / movingTargets);
    445488              if (showMarkers) {
    446489                dp.MarkerStyle = MarkerStyle.Circle;
     
    470513          if (row.Points.Count > 0 && row.Points.Last().XValue == h.Key) {
    471514            row.Points.Last().SetValueY(ecdf / movingTargets);
    472             row.Points.Last().Color = Color.FromArgb(255 - (int)Math.Floor(255 * (missedecdf / totalTargets)), colors[colorCount]);
    473515          } else {
    474             var dp = new DataPoint(h.Key, ecdf / movingTargets) {
    475               Color = Color.FromArgb(255 - (int)Math.Floor(255 * (missedecdf / totalTargets)), colors[colorCount])
    476             };
     516            var dp = new DataPoint(h.Key, ecdf / movingTargets);
    477517            if (showMarkers) {
    478518              dp.MarkerStyle = MarkerStyle.Circle;
     
    492532          // if there are misses beyond the last hit we extend the shaded area
    493533          missedecdf += iter.Current.Value;
    494           var dp = new DataPoint(iter.Current.Key, ecdf / movingTargets) {
    495             Color = Color.FromArgb(255 - (int)Math.Floor(255 * (missedecdf / totalTargets)), colors[colorCount])
    496           };
    497           if (showMarkers) {
    498             dp.MarkerStyle = MarkerStyle.Circle;
    499             dp.MarkerBorderWidth = 1;
    500             dp.MarkerSize = 5;
    501           }
    502           row.Points.Add(dp);
    503           if (boundShadingCheckBox.Checked) {
    504             rowShade.Points.Add(new DataPoint(iter.Current.Key, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
     534          if (row.Points.Count == 0 || row.Points.Last().XValue < iter.Current.Key) {
     535            var dp = new DataPoint(iter.Current.Key, ecdf / movingTargets);
     536            if (showMarkers) {
     537              dp.MarkerStyle = MarkerStyle.Circle;
     538              dp.MarkerBorderWidth = 1;
     539              dp.MarkerSize = 5;
     540            }
     541            row.Points.Add(dp);
     542            if (boundShadingCheckBox.Checked) {
     543              rowShade.Points.Add(new DataPoint(iter.Current.Key, new[] { ecdf / totalTargets, (ecdf + missedecdf) / totalTargets }));
     544            }
    505545          }
    506546          moreMisses = iter.MoveNext();
     
    517557        }
    518558
    519         if (!labelPrinted) {
     559        if (!labelPrinted && row.Points.Count > 0) {
    520560          var point = row.Points.Last();
    521561          if (showLabelsInTargetChart)
     
    554594    private IEnumerable<double> GetAbsoluteTargets(ProblemInstance pInstance) {
    555595      if (!targetsAreRelative) return targets;
    556       var maximization = pInstance.IsMaximization();
     596      if (!pInstance.Maximization.HasValue) throw new ArgumentException("Problem doesn't specify if it is to be maximized or minimized.");
     597
     598      var maximization = pInstance.Maximization.Value;
    557599      var bestKnown = pInstance.BestKnownQuality;
    558600      if (double.IsNaN(bestKnown)) throw new ArgumentException("Problem instance does not have a defined best - known quality.");
     
    570612
    571613    private double[] GetAbsoluteTargetsWorstToBest(ProblemInstance pInstance) {
    572       if (double.IsNaN(pInstance.BestKnownQuality)) throw new ArgumentException("Problem instance does not have a defined best-known quality.");
     614      if (targetsAreRelative && double.IsNaN(pInstance.BestKnownQuality)) throw new ArgumentException("Problem instance does not have a defined best-known quality.");
     615      if (!pInstance.Maximization.HasValue) throw new ArgumentException("Problem doesn't specify if it is to be maximized or minimized.");
    573616      var absTargets = GetAbsoluteTargets(pInstance);
    574       return (pInstance.IsMaximization()
     617      return (pInstance.Maximization.Value
    575618        ? absTargets.OrderBy(x => x) : absTargets.OrderByDescending(x => x)).ToArray();
    576619    }
     
    582625
    583626    private Tuple<bool, double> GetEffortToHitTarget(
    584         ObservableList<Tuple<double, double>> convergenceGraph,
     627        ConvergenceGraph cgraph,
    585628        double absTarget, bool maximization) {
    586       if (convergenceGraph.Count == 0)
     629      if (cgraph.Points == 0)
    587630        throw new ArgumentException("Convergence graph is empty.", "convergenceGraph");
    588631     
    589       var index = convergenceGraph.BinarySearch(Tuple.Create(0.0, absTarget), new TargetComparer(maximization));
     632      var index = cgraph.BinarySearch(new ConvergenceGraphPoint(0.0, absTarget), new TargetComparer(maximization));
    590633      if (index >= 0) {
    591         return Tuple.Create(true, convergenceGraph[index].Item1);
     634        return Tuple.Create(true, cgraph[index].Runlength);
    592635      } else {
    593636        index = ~index;
    594         if (index >= convergenceGraph.Count)
    595           return Tuple.Create(false, convergenceGraph.Last().Item1);
    596         return Tuple.Create(true, convergenceGraph[index].Item1);
    597       }
    598     }
    599 
    600     private void UpdateErtTables(List<AlgorithmInstance> algorithmInstances) {
     637        if (index >= cgraph.Points)
     638          return Tuple.Create(false, cgraph.Last().Runlength);
     639        return Tuple.Create(true, cgraph[index].Runlength);
     640      }
     641    }
     642
     643    private void UpdateErtTables() {
    601644      ertTableView.Content = null;
    602645      var columns = targets.Length + 1;
    603       var totalRows = algorithmInstances.Count * algorithmInstances.Max(x => x.GetNumberOfProblemInstances()) + algorithmInstances.Max(x => x.GetNumberOfProblemInstances());
     646      var totalRows = groups.Count * groups.Max(x => x.GetNumberOfCases()) + groups.Max(x => x.GetNumberOfCases());
    604647      var matrix = new StringMatrix(totalRows, columns);
    605648      var rowNames = new List<string>();
     
    611654      if (string.IsNullOrEmpty(tableName)) return;
    612655     
    613       var problems = algorithmInstances.SelectMany(x => x.GetProblemInstances()).Distinct().ToList();
    614 
    615       foreach (var problem in problems) {
    616         var max = problem.IsMaximization();
     656      var problems = groups.SelectMany(x => x.GetCases()).Distinct().ToList();
     657
     658      foreach (var problem in problems.OrderBy(x => x.ProblemName, new NaturalStringComparer())) {
     659        if (double.IsNaN(problem.BestKnownQuality) || !problem.Maximization.HasValue) continue;
    617660        rowNames.Add(problem.ToString());
     661        var max = problem.Maximization.Value;
    618662        var absTargets = GetAbsoluteTargetsWorstToBest(problem);
    619663        if (targetsAreRelative) {
     
    625669        rowCount++;
    626670
    627         foreach (var alg in algorithmInstances) {
     671        foreach (var alg in groups) {
    628672          rowNames.Add(alg.Name);
    629           var runs = alg.GetRuns(problem).ToList();
     673          var runs = alg.GetTrials(problem).ToList();
    630674          if (runs.Count == 0) {
    631675            matrix[rowCount, columns - 1] = "N/A";
     
    635679          var result = default(ErtCalculationResult);
    636680          for (var i = 0; i < absTargets.Length; i++) {
    637             result = ExpectedRuntimeHelper.CalculateErt(runs, tableName, absTargets[i], max);
     681            result = ExpectedRuntimeHelper.CalculateErt(runs.Select(x => x.ToTuples()), absTargets[i], max);
    638682            matrix[rowCount, i] = result.ToString();
    639683          }
     
    658702
    659703      if (budgets == null) GenerateDefaultBudgets(table);
    660 
    661       var algInstances = GroupRuns().ToList();
    662       if (algInstances.Count == 0) return;
     704     
     705      if (groups.Count == 0) return;
    663706
    664707      var colorCount = 0;
    665708      var lineStyleCount = 0;
    666709     
    667       foreach (var alg in algInstances) {
     710      foreach (var alg in groups) {
    668711        var hits = new Dictionary<string, SortedList<double, double>>();
    669712
    670         foreach (var problem in alg.GetProblemInstances()) {
    671           foreach (var run in alg.GetRuns(problem)) {
    672             var resultsTable = (IndexedDataTable<double>)run.Results[table];
    673 
    674             CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), problem, alg.Name);
     713        foreach (var problem in alg.GetCases()) {
     714          foreach (var run in alg.GetTrials(problem)) {
     715            CalculateHitsForEachBudget(hits, run, problem, alg.Name);
    675716          }
    676717        }
     
    705746
    706747    private void GenerateDefaultBudgets(string table) {
    707       var runs = Content;
    708       var min = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Min()).Min();
    709       var max = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Max()).Max();
     748      var runs = Content.Where(x => x.Results.ContainsKey(table) && x.Results[table] is IndexedDataTable<double>)
     749                        .Select(x => (IndexedDataTable<double>)x.Results[table])
     750                        .Where(x => x.Rows.Count > 0 && x.Rows.First().Values.Count > 0)
     751                        .Select(x => x.Rows.First())
     752                        .ToList();
     753      if (runs.Count == 0) {
     754        budgets = new double[0];
     755        suppressBudgetsEvents = true;
     756        budgetsTextBox.Text = string.Empty;
     757        suppressBudgetsEvents = false;
     758        return;
     759      }
     760
     761      var min = runs.Select(x => x.Values.Select(y => y.Item1).Min()).Min();
     762      var max = runs.Select(x => x.Values.Select(y => y.Item1).Max()).Max();
    710763      var points = 3;
    711764      budgets = Enumerable.Range(1, points).Select(x => min + (x / (double)points) * (max - min)).ToArray();
     
    715768    }
    716769
    717     private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, ProblemInstance problem, string groupName) {
    718       var max = problem.IsMaximization();
     770    private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, ConvergenceGraph cgraph, ProblemInstance problem, string groupName) {
     771      var max = problem.Maximization.Value;
    719772      var prevIndex = 0;
    720773      foreach (var b in budgets) {
    721774        var key = groupName + "-" + b;
    722         var index = row.Values.BinarySearch(prevIndex, row.Values.Count - prevIndex, Tuple.Create(b, 0.0), new CostComparer());
     775        var index = cgraph.BinarySearch(prevIndex, cgraph.Points - prevIndex, new ConvergenceGraphPoint(b, 0.0), new CostComparer());
    723776        if (index < 0) {
    724777          index = ~index;
    725           if (index >= row.Values.Count) break; // the run wasn't long enough to use up budget b (or any subsequent larger one)
     778          if (index >= cgraph.Points) break; // the run wasn't long enough to use up budget b (or any subsequent larger one)
    726779        }
    727780        if (!hits.ContainsKey(key)) hits.Add(key, new SortedList<double, double>());
    728         var v = row.Values[index];
    729         var relTgt = CalculateRelativeDifference(max, problem.BestKnownQuality, v.Item2) + 1;
     781        var v = cgraph[index];
     782        var relTgt = CalculateRelativeDifference(max, problem.BestKnownQuality, v.Quality) + 1;
    730783        if (hits[key].ContainsKey(relTgt))
    731784          hits[key][relTgt]++;
     
    743796      if (InvokeRequired) Invoke((Action)SynchronizeTargetTextBox);
    744797      else {
    745         suppressTargetsEvents = true;
    746         try {
    747           if (targetsAreRelative)
    748             targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100)) + "%";
    749           else targetsTextBox.Text = string.Join(" ; ", targets);
    750         } finally { suppressTargetsEvents = false; }
     798        if (targetsAreRelative)
     799          targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100)) + "%";
     800        else targetsTextBox.Text = string.Join(" ; ", targets);
    751801      }
    752802    }
    753803
    754804    private void groupComboBox_SelectedIndexChanged(object sender, EventArgs e) {
    755       UpdateRuns();
    756       SetEnabledStateOfControls();
     805      if (updateInProgress) return;
     806      try {
     807        updateInProgress = true;
     808        UpdateRuns();
     809        SetEnabledStateOfControls();
     810      } finally { updateInProgress = false; }
    757811    }
    758812    private void problemComboBox_SelectedIndexChanged(object sender, EventArgs e) {
    759       UpdateRuns();
    760       SetEnabledStateOfControls();
     813      if (updateInProgress) return;
     814      try {
     815        updateInProgress = true;
     816        UpdateRuns();
     817        SetEnabledStateOfControls();
     818      } finally { updateInProgress = false; }
    761819    }
    762820    private void dataTableComboBox_SelectedIndexChanged(object sender, EventArgs e) {
    763       if (dataTableComboBox.SelectedIndex >= 0)
    764         GenerateDefaultBudgets((string)dataTableComboBox.SelectedItem);
    765       UpdateBestKnownQualities();
    766       UpdateRuns();
    767       SetEnabledStateOfControls();
     821      if (updateInProgress) return;
     822      try {
     823        updateInProgress = true;
     824        if (dataTableComboBox.SelectedIndex >= 0)
     825          GenerateDefaultBudgets((string)dataTableComboBox.SelectedItem);
     826        UpdateBestKnownQualities();
     827        UpdateRuns();
     828        SetEnabledStateOfControls();
     829      } finally { updateInProgress = false; }
    768830    }
    769831
     
    778840
    779841    #region Event handlers for target analysis
    780     private bool suppressTargetsEvents;
    781842    private void targetsTextBox_Validating(object sender, CancelEventArgs e) {
    782       if (suppressTargetsEvents) return;
    783       var targetStrings = targetsTextBox.Text.Split(new[] { '%', ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
    784       var targetList = new List<decimal>();
    785       foreach (var ts in targetStrings) {
    786         decimal t;
    787         if (!decimal.TryParse(ts, out t)) {
    788           errorProvider.SetError(targetsTextBox, "Not all targets can be parsed: " + ts);
     843      if (updateInProgress) return;
     844      try {
     845        updateInProgress = true;
     846        var targetStrings = targetsTextBox.Text.Split(new[] { '%', ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
     847        var targetList = new List<decimal>();
     848        foreach (var ts in targetStrings) {
     849          decimal t;
     850          if (!decimal.TryParse(ts, out t)) {
     851            errorProvider.SetError(targetsTextBox, "Not all targets can be parsed: " + ts);
     852            e.Cancel = true;
     853            return;
     854          }
     855          if (targetsAreRelative)
     856            targetList.Add(t / 100);
     857          else targetList.Add(t);
     858        }
     859        if (targetList.Count == 0) {
     860          errorProvider.SetError(targetsTextBox, "Give at least one target value!");
    789861          e.Cancel = true;
    790862          return;
    791863        }
    792         if (targetsAreRelative)
    793           targetList.Add(t / 100);
    794         else targetList.Add(t);
    795       }
    796       if (targetList.Count == 0) {
    797         errorProvider.SetError(targetsTextBox, "Give at least one target value!");
    798         e.Cancel = true;
    799         return;
    800       }
    801       e.Cancel = false;
    802       errorProvider.SetError(targetsTextBox, null);
    803       targets = targetsAreRelative ? targetList.Select(x => (double)x).OrderByDescending(x => x).ToArray() : targetList.Select(x => (double)x).ToArray();
    804 
    805       SynchronizeTargetTextBox();
    806       UpdateResultsByTarget();
    807       SetEnabledStateOfControls();
     864        e.Cancel = false;
     865        errorProvider.SetError(targetsTextBox, null);
     866        targets = targetsAreRelative ? targetList.Select(x => (double)x).OrderByDescending(x => x).ToArray() : targetList.Select(x => (double)x).ToArray();
     867
     868        SynchronizeTargetTextBox();
     869        UpdateResultsByTarget();
     870        SetEnabledStateOfControls();
     871
     872      } finally { updateInProgress = false; }
    808873    }
    809874
    810875    private void aggregateTargetsCheckBox_CheckedChanged(object sender, EventArgs e) {
    811       SuspendRepaint();
     876      if (updateInProgress) return;
    812877      try {
    813         UpdateResultsByTarget();
    814       } finally { ResumeRepaint(true); }
    815     }
    816 
    817     private void relativeOrAbsoluteComboBox_SelectedIndexChanged(object sender, EventArgs e) {
    818       if (suppressUpdates) return;
    819       var pd = (ProblemInstance)problemComboBox.SelectedItem;
    820       if (!double.IsNaN(pd.BestKnownQuality)) {
    821         var max = pd.IsMaximization();
    822         if (targetsAreRelative) targets = GetAbsoluteTargets(pd).ToArray();
    823         else {
    824           // Rounding to 5 digits since it's certainly appropriate for this application
    825           if (pd.BestKnownQuality > 0) {
    826             targets = targets.Select(x => Math.Round(max ? 1.0 - (x / pd.BestKnownQuality) : (x / pd.BestKnownQuality) - 1.0, 5)).ToArray();
    827           } else if (pd.BestKnownQuality < 0) {
    828             targets = targets.Select(x => Math.Round(!max ? 1.0 - (x / pd.BestKnownQuality) : (x / pd.BestKnownQuality) - 1.0, 5)).ToArray();
    829           }
    830         }
    831       }
    832       targetsAreRelative = (string)relativeOrAbsoluteComboBox.SelectedItem == "relative";
    833       SynchronizeTargetTextBox();
    834 
    835       try {
     878        updateInProgress = true;
    836879        SuspendRepaint();
    837880        UpdateResultsByTarget();
    838       } finally { ResumeRepaint(true); }
     881      } finally {
     882        updateInProgress = false;
     883        ResumeRepaint(true);
     884      }
     885    }
     886
     887    private void relativeOrAbsoluteComboBox_SelectedIndexChanged(object sender, EventArgs e) {
     888      if (updateInProgress) return;
     889      try {
     890        updateInProgress = true;
     891        var pd = (ProblemInstance)problemComboBox.SelectedItem;
     892        if (!double.IsNaN(pd.BestKnownQuality) && pd.Maximization.HasValue) {
     893          var max = pd.Maximization.Value;
     894          if (targetsAreRelative) targets = GetAbsoluteTargets(pd).ToArray();
     895          else {
     896            // Rounding to 5 digits since it's certainly appropriate for this application
     897            if (pd.BestKnownQuality > 0) {
     898              targets = targets.Select(x => Math.Round(max ? 1.0 - (x / pd.BestKnownQuality) : (x / pd.BestKnownQuality) - 1.0, 5)).ToArray();
     899            } else if (pd.BestKnownQuality < 0) {
     900              targets = targets.Select(x => Math.Round(!max ? 1.0 - (x / pd.BestKnownQuality) : (x / pd.BestKnownQuality) - 1.0, 5)).ToArray();
     901            }
     902          }
     903        }
     904        targetsAreRelative = (string)relativeOrAbsoluteComboBox.SelectedItem == "relative";
     905        SynchronizeTargetTextBox();
     906       
     907        SuspendRepaint();
     908        UpdateResultsByTarget();
     909      } finally {
     910        updateInProgress = false;
     911        ResumeRepaint(true);
     912      }
    839913    }
    840914
     
    856930              : dialog.Values.Select(x => (double)x).ToArray();
    857931
    858             SynchronizeTargetTextBox();
    859             UpdateResultsByTarget();
    860             SetEnabledStateOfControls();
     932            try {
     933              updateInProgress = true;
     934              SynchronizeTargetTextBox();
     935              UpdateResultsByTarget();
     936              SetEnabledStateOfControls();
     937            } finally { updateInProgress = false; }
    861938          }
    862939        }
     
    869946     
    870947      foreach (var run in Content) {
    871         if (!run.Results.ContainsKey(table)) continue;
     948        if (!run.Results.ContainsKey(table) || !(run.Results[table] is IndexedDataTable<double>)) continue;
    872949        var resultsTable = (IndexedDataTable<double>)run.Results[table];
    873950        var values = resultsTable.Rows.First().Values;
    874951        var pd = new ProblemInstance(run);
     952        if (!pd.Maximization.HasValue) continue;
    875953        pd = problems.Single(x => x.Equals(pd));
    876         var max = pd.IsMaximization();
     954        if (targetsAreRelative && double.IsNaN(pd.BestKnownQuality)) continue;
     955
     956        var max = pd.Maximization.Value;
    877957        var absTargets = GetAbsoluteTargetsWorstToBest(pd);
     958        var cgraph = new ConvergenceGraph(resultsTable, max);
    878959
    879960        var prevIndex = 0;
    880961        for (var i = 0; i < absTargets.Length; i++) {
    881962          var absTarget = absTargets[i];
    882           var index = values.BinarySearch(prevIndex, values.Count - prevIndex, Tuple.Create(0.0, absTarget), new TargetComparer(max));
     963          var index = cgraph.BinarySearch(prevIndex, values.Count - prevIndex, new ConvergenceGraphPoint(0.0, absTarget), new TargetComparer(max));
    883964          if (index < 0) {
    884965            index = ~index;
     
    893974
    894975    private void markerCheckBox_CheckedChanged(object sender, EventArgs e) {
    895       SuspendRepaint();
    896976      try {
     977        updateInProgress = true;
     978        SuspendRepaint();
    897979        UpdateResultsByTarget();
    898       } finally { ResumeRepaint(true); }
     980      } finally {
     981        updateInProgress = false;
     982        ResumeRepaint(true);
     983      }
    899984    }
    900985
    901986    private void showLabelsCheckBox_CheckedChanged(object sender, EventArgs e) {
    902987      showLabelsInTargetChart = showLabelsCheckBox.Checked;
    903       SuspendRepaint();
    904988      try {
     989        updateInProgress = true;
     990        SuspendRepaint();
    905991        UpdateResultsByTarget();
    906       } finally { ResumeRepaint(true); }
     992      } finally {
     993        updateInProgress = false;
     994        ResumeRepaint(true);
     995      }
    907996    }
    908997    #endregion
     
    9761065        var values = resultsTable.Rows.First().Values;
    9771066        var pd = new ProblemInstance(run);
     1067        if (!pd.Maximization.HasValue) continue;
    9781068        pd = problems.Single(x => x.Equals(pd));
    979 
     1069        var max = pd.Maximization.Value;
     1070        var cgraph = new ConvergenceGraph(resultsTable, max);
    9801071        var prevIndex = 0;
    9811072        foreach (var b in budgets) {
    982           var index = values.BinarySearch(prevIndex, values.Count - prevIndex, Tuple.Create(b, 0.0), new CostComparer());
     1073          var index = cgraph.BinarySearch(prevIndex, values.Count - prevIndex, new ConvergenceGraphPoint(b, 0.0), new CostComparer());
    9831074          if (index < 0) {
    9841075            index = ~index;
     
    9861077          }
    9871078          var v = values[index];
    988           var tgt = targetsAreRelative ? CalculateRelativeDifference(pd.IsMaximization(), pd.BestKnownQuality, v.Item2) : v.Item2;
     1079          var tgt = targetsAreRelative ? CalculateRelativeDifference(max, pd.BestKnownQuality, v.Item2) : v.Item2;
    9891080          run.Results[table + (targetsAreRelative ? ".CostForRelTarget " : ".CostForAbsTarget ") + b] = new DoubleValue(tgt);
    9901081          prevIndex = index;
     
    10111102    private Dictionary<ProblemInstance, double> CalculateBestTargetPerProblemInstance(string table) {
    10121103      if (table == null) table = string.Empty;
    1013       return (from r in Content
    1014               where r.Visible
    1015               let pd = new ProblemInstance(r)
    1016               let target = r.Parameters.ContainsKey("BestKnownQuality")
    1017                            && r.Parameters["BestKnownQuality"] is DoubleValue
    1018                 ? ((DoubleValue)r.Parameters["BestKnownQuality"]).Value
    1019                 : (r.Results.ContainsKey(table) && r.Results[table] is IndexedDataTable<double>
    1020                   ? ((IndexedDataTable<double>)r.Results[table]).Rows.First().Values.Last().Item2
    1021                   : double.NaN)
    1022               group target by pd into g
    1023               select new { Problem = g.Key, Target = g.Any(x => !double.IsNaN(x)) ? (g.Key.IsMaximization() ? g.Max() : g.Where(x => !double.IsNaN(x)).Min()) : double.NaN })
    1024         .ToDictionary(x => x.Problem, x => x.Target);
     1104      var dict = new Dictionary<ProblemInstance, double>();
     1105      foreach (var k in from r in Content where r.Visible let pd = new ProblemInstance(r) group r by pd into g select new { Problem = g.Key, Runs = g.ToList() }) {
     1106        var pd = k.Problem;
     1107        if (!pd.Maximization.HasValue) continue;
     1108        var values = GetQualityValues(k.Runs, table).ToList();
     1109        var target = double.NaN;
     1110        if (values.Count > 0) {
     1111          target = pd.Maximization.Value ? values.Max() : values.Min();
     1112        }
     1113        dict[pd] = target;
     1114      }
     1115      return dict;
     1116    }
     1117
     1118    private IEnumerable<double> GetQualityValues(List<IRun> runs, string table) {
     1119      foreach (var r in runs) {
     1120        IItem item;
     1121        if (r.Parameters.TryGetValue("BestKnownQuality", out item)) {
     1122          var dval = item as DoubleValue;
     1123          if (dval != null && !double.IsNaN(dval.Value))
     1124            yield return dval.Value;
     1125        }
     1126        if (r.Results.TryGetValue(table, out item)) {
     1127          var dt = item as IndexedDataTable<double>;
     1128          if (dt != null && dt.Rows.Count > 0 && dt.Rows.First().Values.Count > 0) {
     1129            var last = dt.Rows.First().Values.Last().Item2;
     1130            if (!double.IsNaN(last))
     1131              yield return Math.Round(last, 10);
     1132          }
     1133        }
     1134      }
    10251135    }
    10261136
     
    10321142      SuspendRepaint();
    10331143      try {
     1144        GroupRuns();
    10341145        UpdateResultsByTarget();
    10351146        UpdateResultsByCost();
     
    11361247    }
    11371248
     1249    #region Helper classes
    11381250    private class AlgorithmInstance : INotifyPropertyChanged {
    11391251      private string name;
     
    11471259      }
    11481260
    1149       private Dictionary<ProblemInstance, List<IRun>> performanceData;
    1150 
    1151       public int GetNumberOfProblemInstances() {
     1261      private Dictionary<ProblemInstance, List<ConvergenceGraph>> performanceData;
     1262
     1263      public int GetNumberOfCases() {
    11521264        return performanceData.Count;
    11531265      }
    11541266
    1155       public IEnumerable<ProblemInstance> GetProblemInstances() {
     1267      public IEnumerable<ProblemInstance> GetCases() {
    11561268        return performanceData.Keys;
    11571269      }
    11581270
    1159       public int GetNumberOfRuns(ProblemInstance p) {
     1271      public int GetNumberOfTrials(ProblemInstance p) {
    11601272        if (p == ProblemInstance.MatchAll) return performanceData.Select(x => x.Value.Count).Sum();
    1161         List<IRun> runs;
    1162         if (performanceData.TryGetValue(p, out runs))
    1163           return runs.Count;
     1273        List<ConvergenceGraph> trials;
     1274        if (performanceData.TryGetValue(p, out trials))
     1275          return trials.Count;
    11641276
    11651277        return 0;
    11661278      }
    11671279
    1168       public IEnumerable<IRun> GetRuns(ProblemInstance p) {
     1280      public IEnumerable<ConvergenceGraph> GetTrials(ProblemInstance p) {
    11691281        if (p == ProblemInstance.MatchAll) return performanceData.SelectMany(x => x.Value);
    1170         List<IRun> runs;
    1171         if (performanceData.TryGetValue(p, out runs))
    1172           return runs;
    1173 
    1174         return Enumerable.Empty<IRun>();
    1175       }
    1176 
    1177       public AlgorithmInstance(string name, IEnumerable<IRun> runs, IEnumerable<ProblemInstance> problems) {
     1282        List<ConvergenceGraph> trials;
     1283        if (performanceData.TryGetValue(p, out trials))
     1284          return trials;
     1285
     1286        return Enumerable.Empty<ConvergenceGraph>();
     1287      }
     1288
     1289      public AlgorithmInstance(string name, IEnumerable<IGrouping<ProblemInstance, IndexedDataTable<double>>> trials) {
    11781290        this.name = name;
    11791291
    1180         var pDict = problems.ToDictionary(r => r, _ => new List<IRun>());
    1181         foreach (var y in runs) {
    1182           var pd = new ProblemInstance(y);
    1183           List<IRun> l;
    1184           if (pDict.TryGetValue(pd, out l)) l.Add(y);
    1185         }
    1186         performanceData = pDict.Where(x => x.Value.Count > 0).ToDictionary(x => x.Key, x => x.Value);
     1292        performanceData = new Dictionary<ProblemInstance, List<ConvergenceGraph>>();
     1293        foreach (var t in trials) {
     1294          if (double.IsNaN(t.Key.BestKnownQuality) || !t.Key.Maximization.HasValue) continue;
     1295          performanceData[t.Key] = t.Select(c => new ConvergenceGraph(c, t.Key.Maximization.Value)).ToList();
     1296        }
    11871297      }
    11881298
     
    12141324        ProblemName = string.Empty;
    12151325        Evaluator = string.Empty;
    1216         Maximization = string.Empty;
     1326        Maximization = null;
    12171327        DisplayProblemType = false;
    12181328        DisplayProblemName = false;
     
    12271337        ProblemName = GetStringValueOrEmpty(run, "Problem Name");
    12281338        Evaluator = GetStringValueOrEmpty(run, "Evaluator");
    1229         Maximization = GetMaximizationValueOrEmpty(run, "Maximization");
     1339        Maximization = GetBoolValueOrEmpty(run, "Maximization");
    12301340        DisplayProblemType = !string.IsNullOrEmpty(ProblemType);
    12311341        DisplayProblemName = !string.IsNullOrEmpty(ProblemName);
    12321342        DisplayEvaluator = !string.IsNullOrEmpty(Evaluator);
    1233         DisplayMaximization = !string.IsNullOrEmpty(Maximization);
     1343        DisplayMaximization = Maximization.HasValue;
    12341344        matchAll = false;
    12351345        BestKnownQuality = GetDoubleValueOrNaN(run, "BestKnownQuality");
     
    12991409        }
    13001410      }
    1301       private string maximization;
    1302       public string Maximization {
     1411      private bool? maximization;
     1412      public bool? Maximization {
    13031413        get { return maximization; }
    13041414        set {
     
    13181428      }
    13191429
    1320       public bool IsMaximization() {
    1321         return Maximization == "MAX";
    1322       }
    1323 
    13241430      public bool Match(IRun run) {
    13251431        return matchAll ||
    1326                GetStringValueOrEmpty(run, "Problem Type") == ProblemType
     1432               (GetStringValueOrEmpty(run, "Problem Type") == ProblemType
    13271433               && GetStringValueOrEmpty(run, "Problem Name") == ProblemName
    13281434               && GetStringValueOrEmpty(run, "Evaluator") == Evaluator
    1329                && GetMaximizationValueOrEmpty(run, "Maximization") == Maximization;
     1435               && GetBoolValueOrEmpty(run, "Maximization") == Maximization);
    13301436      }
    13311437
     
    13481454      }
    13491455
    1350       private string GetMaximizationValueOrEmpty(IRun run, string key) {
     1456      private bool? GetBoolValueOrEmpty(IRun run, string key) {
    13511457        IItem param;
    13521458        if (run.Parameters.TryGetValue(key, out param)) {
    13531459          var bv = param as BoolValue;
    1354           return bv != null ? (bv.Value ? "MAX" : "MIN") : string.Empty;
    1355         }
    1356         return string.Empty;
     1460          if (bv != null) return bv.Value;
     1461        }
     1462        return null;
    13571463      }
    13581464
     
    13751481          (DisplayProblemName ? ProblemName : string.Empty),
    13761482          (DisplayEvaluator ? Evaluator : string.Empty),
    1377           (DisplayMaximization ? Maximization : string.Empty),
     1483          (DisplayMaximization && Maximization.HasValue ? (Maximization.Value ? "MAX" : "MIN") : string.Empty),
    13781484          !double.IsNaN(BestKnownQuality) ? BestKnownQuality.ToString(CultureInfo.CurrentCulture.NumberFormat) : string.Empty }.Where(x => !string.IsNullOrEmpty(x)));
    13791485      }
     
    13861492    }
    13871493
    1388     private class CostComparer : Comparer<Tuple<double, double>> {
    1389       public override int Compare(Tuple<double, double> x, Tuple<double, double> y) {
    1390         return x.Item1.CompareTo(y.Item1);
    1391       }
    1392     }
    1393 
    1394     private class TargetComparer : Comparer<Tuple<double, double>> {
     1494    private class CostComparer : Comparer<ConvergenceGraphPoint> {
     1495      public override int Compare(ConvergenceGraphPoint x, ConvergenceGraphPoint y) {
     1496        return x.Runlength.CompareTo(y.Runlength);
     1497      }
     1498    }
     1499
     1500    private class TargetComparer : Comparer<ConvergenceGraphPoint> {
    13951501      public bool Maximization { get; private set; }
    13961502      public TargetComparer(bool maximization) {
     
    13981504      }
    13991505
    1400       public override int Compare(Tuple<double, double> x, Tuple<double, double> y) {
    1401         return Maximization ? x.Item2.CompareTo(y.Item2) : y.Item2.CompareTo(x.Item2);
    1402       }
    1403     }
     1506      public override int Compare(ConvergenceGraphPoint x, ConvergenceGraphPoint y) {
     1507        return Maximization ? x.Quality.CompareTo(y.Quality) : y.Quality.CompareTo(x.Quality);
     1508      }
     1509    }
     1510
     1511    private class ConvergenceGraph : IEnumerable<ConvergenceGraphPoint> {
     1512      private List<ConvergenceGraphPoint> data;
     1513      private bool maximization;
     1514      private string xAxisLabel;
     1515
     1516      public string XAxisName { get { return xAxisLabel; } }
     1517
     1518      public int Points {
     1519        get { return data.Count; }
     1520      }
     1521
     1522      public double TotalRunlength {
     1523        get { return data.Last().Runlength; }
     1524      }
     1525
     1526      public double BestQuality {
     1527        get { return data.Last().Quality; }
     1528      }
     1529
     1530      public double QualityAt(double runlength) {
     1531        var point = data.SkipWhile(x => x.Runlength < runlength).FirstOrDefault();
     1532        if (point == null) return double.NaN;
     1533        return point.Quality;
     1534      }
     1535
     1536      public double RunlengthFor(double quality) {
     1537        var point = (maximization ? data.SkipWhile(x => x.Quality < quality) : data.SkipWhile(x => x.Quality > quality)).FirstOrDefault();
     1538        if (point == null) return double.NaN;
     1539        return point.Runlength;
     1540      }
     1541
     1542      public ConvergenceGraphPoint this[int point] {
     1543        get { return data[point]; }
     1544      }
     1545
     1546      public ConvergenceGraph(IndexedDataTable<double> table, bool maximization) {
     1547        data = table.Rows.First().Values.Select(x => new ConvergenceGraphPoint() { Runlength = x.Item1, Quality = x.Item2 }).ToList();
     1548        xAxisLabel = table.VisualProperties.XAxisTitle;
     1549        this.maximization = maximization;
     1550      }
     1551
     1552      public IEnumerator<ConvergenceGraphPoint> GetEnumerator() {
     1553        return data.GetEnumerator();
     1554      }
     1555
     1556      IEnumerator IEnumerable.GetEnumerator() {
     1557        return data.GetEnumerator();
     1558      }
     1559
     1560      public int BinarySearch(ConvergenceGraphPoint item, IComparer<ConvergenceGraphPoint> comparer) {
     1561        return data.BinarySearch(item, comparer);
     1562      }
     1563
     1564      public int BinarySearch(int index, int count, ConvergenceGraphPoint point, IComparer<ConvergenceGraphPoint> comparer) {
     1565        return data.BinarySearch(index, count, point, comparer);
     1566      }
     1567
     1568      public IEnumerable<Tuple<double, double>> ToTuples() {
     1569        return data.Select(x => Tuple.Create(x.Runlength, x.Quality));
     1570      }
     1571    }
     1572
     1573    private class ConvergenceGraphPoint {
     1574      public double Runlength { get; set; }
     1575      public double Quality { get; set; }
     1576
     1577      public ConvergenceGraphPoint() {
     1578
     1579      }
     1580      public ConvergenceGraphPoint(double runlength, double quality) {
     1581        Runlength = runlength;
     1582        Quality = quality;
     1583      }
     1584    }
     1585    #endregion
    14041586  }
    14051587}
Note: See TracChangeset for help on using the changeset viewer.