Changeset 14665


Ignore:
Timestamp:
02/11/17 01:05:41 (4 years ago)
Author:
abeham
Message:

#2457: updated branch to trunk

Location:
branches/PerformanceComparison
Files:
7 edited
2 copied

Legend:

Unmodified
Added
Removed
  • branches/PerformanceComparison/HeuristicLab.Analysis

    • Property svn:mergeinfo changed (with no actual effect on merging)
  • branches/PerformanceComparison/HeuristicLab.Analysis.Views

    • Property svn:mergeinfo changed (with no actual effect on merging)
  • branches/PerformanceComparison/HeuristicLab.Optimization

    • Property svn:mergeinfo changed (with no actual effect on merging)
  • branches/PerformanceComparison/HeuristicLab.Optimization.Views

  • branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/HeuristicLab.Optimization.Views-3.3.csproj

    r13737 r14665  
    349349      <DependentUpon>RunCollectionEqualityConstraintView.cs</DependentUpon>
    350350    </Compile>
     351    <Compile Include="RunCollectionViews\RunCollectionParameterAnalysisView.cs">
     352      <SubType>UserControl</SubType>
     353    </Compile>
     354    <Compile Include="RunCollectionViews\RunCollectionParameterAnalysisView.Designer.cs">
     355      <DependentUpon>RunCollectionParameterAnalysisView.cs</DependentUpon>
     356    </Compile>
    351357    <Compile Include="RunCollectionViews\RunCollectionRLDView.cs">
    352358      <SubType>UserControl</SubType>
  • branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.Designer.cs

    r14101 r14665  
    6464      this.tabControl = new System.Windows.Forms.TabControl();
    6565      this.byTargetTabPage = new System.Windows.Forms.TabPage();
     66      this.relativeOrAbsoluteComboBox = new System.Windows.Forms.ComboBox();
    6667      this.targetChart = new HeuristicLab.Visualization.ChartControlsExtensions.EnhancedChart();
     68      this.markerCheckBox = new System.Windows.Forms.CheckBox();
    6769      this.boundShadingCheckBox = new System.Windows.Forms.CheckBox();
    6870      this.byCostTabPage = new System.Windows.Forms.TabPage();
    6971      this.byCostViewHost = new HeuristicLab.MainForm.WindowsForms.ViewHost();
    7072      this.budgetLogScalingCheckBox = new System.Windows.Forms.CheckBox();
    71       this.eachOrAllBudgetsCheckBox = new System.Windows.Forms.CheckBox();
     73      this.aggregateBudgetsCheckBox = new System.Windows.Forms.CheckBox();
    7274      this.generateBudgetsButton = new System.Windows.Forms.Button();
    7375      this.byTableTabPage = new System.Windows.Forms.TabPage();
     
    143145      this.targetsTextBox.Location = new System.Drawing.Point(59, 8);
    144146      this.targetsTextBox.Name = "targetsTextBox";
    145       this.targetsTextBox.Size = new System.Drawing.Size(287, 20);
     147      this.targetsTextBox.Size = new System.Drawing.Size(204, 20);
    146148      this.targetsTextBox.TabIndex = 1;
    147149      this.toolTip.SetToolTip(this.targetsTextBox, "The order of the targets is important, first to-hit targets\r\nshould be given firs" +
     
    200202      this.budgetsTextBox.Location = new System.Drawing.Point(59, 8);
    201203      this.budgetsTextBox.Name = "budgetsTextBox";
    202       this.budgetsTextBox.Size = new System.Drawing.Size(326, 20);
     204      this.budgetsTextBox.Size = new System.Drawing.Size(309, 20);
    203205      this.budgetsTextBox.TabIndex = 6;
    204206      this.budgetsTextBox.Validating += new System.ComponentModel.CancelEventHandler(this.budgetsTextBox_Validating);
     
    221223      this.aggregateTargetsCheckBox.Checked = true;
    222224      this.aggregateTargetsCheckBox.CheckState = System.Windows.Forms.CheckState.Checked;
    223       this.aggregateTargetsCheckBox.Location = new System.Drawing.Point(352, 10);
     225      this.aggregateTargetsCheckBox.Location = new System.Drawing.Point(374, 10);
    224226      this.aggregateTargetsCheckBox.Name = "aggregateTargetsCheckBox";
    225227      this.aggregateTargetsCheckBox.Size = new System.Drawing.Size(74, 17);
     
    245247      // byTargetTabPage
    246248      //
     249      this.byTargetTabPage.Controls.Add(this.relativeOrAbsoluteComboBox);
    247250      this.byTargetTabPage.Controls.Add(this.targetChart);
     251      this.byTargetTabPage.Controls.Add(this.markerCheckBox);
    248252      this.byTargetTabPage.Controls.Add(this.boundShadingCheckBox);
    249253      this.byTargetTabPage.Controls.Add(this.targetLogScalingCheckBox);
     
    260264      this.byTargetTabPage.Text = "Performance by Target";
    261265      this.byTargetTabPage.UseVisualStyleBackColor = true;
     266      //
     267      // relativeOrAbsoluteComboBox
     268      //
     269      this.relativeOrAbsoluteComboBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
     270      this.relativeOrAbsoluteComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
     271      this.relativeOrAbsoluteComboBox.FormattingEnabled = true;
     272      this.relativeOrAbsoluteComboBox.Items.AddRange(new object[] {
     273            "relative",
     274            "absolute"});
     275      this.relativeOrAbsoluteComboBox.Location = new System.Drawing.Point(269, 8);
     276      this.relativeOrAbsoluteComboBox.Name = "relativeOrAbsoluteComboBox";
     277      this.relativeOrAbsoluteComboBox.Size = new System.Drawing.Size(99, 21);
     278      this.relativeOrAbsoluteComboBox.TabIndex = 8;
     279      this.relativeOrAbsoluteComboBox.SelectedIndexChanged += new System.EventHandler(this.relativeOrAbsoluteComboBox_SelectedIndexChanged);
    262280      //
    263281      // targetChart
     
    291309      this.targetChart.MouseMove += new System.Windows.Forms.MouseEventHandler(this.chart_MouseMove);
    292310      //
     311      // markerCheckBox
     312      //
     313      this.markerCheckBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Bottom | System.Windows.Forms.AnchorStyles.Left)));
     314      this.markerCheckBox.AutoSize = true;
     315      this.markerCheckBox.Location = new System.Drawing.Point(229, 364);
     316      this.markerCheckBox.Name = "markerCheckBox";
     317      this.markerCheckBox.Size = new System.Drawing.Size(91, 17);
     318      this.markerCheckBox.TabIndex = 6;
     319      this.markerCheckBox.Text = "show markers";
     320      this.markerCheckBox.UseVisualStyleBackColor = true;
     321      this.markerCheckBox.CheckedChanged += new System.EventHandler(this.markerCheckBox_CheckedChanged);
     322      //
    293323      // boundShadingCheckBox
    294324      //
     
    309339      this.byCostTabPage.Controls.Add(this.byCostViewHost);
    310340      this.byCostTabPage.Controls.Add(this.budgetLogScalingCheckBox);
    311       this.byCostTabPage.Controls.Add(this.eachOrAllBudgetsCheckBox);
     341      this.byCostTabPage.Controls.Add(this.aggregateBudgetsCheckBox);
    312342      this.byCostTabPage.Controls.Add(this.generateBudgetsButton);
    313343      this.byCostTabPage.Controls.Add(this.budgetsLabel);
     
    350380      this.budgetLogScalingCheckBox.CheckedChanged += new System.EventHandler(this.logScalingCheckBox_CheckedChanged);
    351381      //
    352       // eachOrAllBudgetsCheckBox
    353       //
    354       this.eachOrAllBudgetsCheckBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
    355       this.eachOrAllBudgetsCheckBox.AutoSize = true;
    356       this.eachOrAllBudgetsCheckBox.Location = new System.Drawing.Point(414, 10);
    357       this.eachOrAllBudgetsCheckBox.Name = "eachOrAllBudgetsCheckBox";
    358       this.eachOrAllBudgetsCheckBox.Size = new System.Drawing.Size(36, 17);
    359       this.eachOrAllBudgetsCheckBox.TabIndex = 10;
    360       this.eachOrAllBudgetsCheckBox.Text = "all";
    361       this.eachOrAllBudgetsCheckBox.UseVisualStyleBackColor = true;
    362       this.eachOrAllBudgetsCheckBox.CheckedChanged += new System.EventHandler(this.eachOrAllBudgetsCheckBox_CheckedChanged);
     382      // aggregateBudgetsCheckBox
     383      //
     384      this.aggregateBudgetsCheckBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
     385      this.aggregateBudgetsCheckBox.AutoSize = true;
     386      this.aggregateBudgetsCheckBox.Checked = true;
     387      this.aggregateBudgetsCheckBox.CheckState = System.Windows.Forms.CheckState.Checked;
     388      this.aggregateBudgetsCheckBox.Location = new System.Drawing.Point(374, 10);
     389      this.aggregateBudgetsCheckBox.Name = "aggregateBudgetsCheckBox";
     390      this.aggregateBudgetsCheckBox.Size = new System.Drawing.Size(74, 17);
     391      this.aggregateBudgetsCheckBox.TabIndex = 10;
     392      this.aggregateBudgetsCheckBox.Text = "aggregate";
     393      this.aggregateBudgetsCheckBox.UseVisualStyleBackColor = true;
     394      this.aggregateBudgetsCheckBox.CheckedChanged += new System.EventHandler(this.aggregateBudgetsCheckBox_CheckedChanged);
    363395      //
    364396      // generateBudgetsButton
     
    464496    private System.Windows.Forms.TabPage byCostTabPage;
    465497    private System.Windows.Forms.TabPage byTableTabPage;
    466     private System.Windows.Forms.CheckBox eachOrAllBudgetsCheckBox;
     498    private System.Windows.Forms.CheckBox aggregateBudgetsCheckBox;
    467499    private System.Windows.Forms.Button generateBudgetsButton;
    468500    private System.Windows.Forms.CheckBox budgetLogScalingCheckBox;
     
    473505    private MainForm.WindowsForms.ViewHost byCostViewHost;
    474506    private System.Windows.Forms.CheckBox boundShadingCheckBox;
     507    private System.Windows.Forms.CheckBox markerCheckBox;
     508    private System.Windows.Forms.ComboBox relativeOrAbsoluteComboBox;
    475509  }
    476510}
  • branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs

    r14101 r14665  
    3030using HeuristicLab.Analysis;
    3131using HeuristicLab.Collections;
     32using HeuristicLab.Core;
    3233using HeuristicLab.Core.Views;
    3334using HeuristicLab.Data;
     
    7980    private double[] targets;
    8081    private double[] budgets;
     82    private bool targetsAreRelative = true;
     83    private readonly BindingList<ProblemInstance> problems;
    8184
    8285    private bool suppressUpdates;
     
    103106      };
    104107      byCostViewHost.Content = byCostDataTable;
     108      suppressUpdates = true;
     109      relativeOrAbsoluteComboBox.SelectedItem = targetsAreRelative ? "relative" : "absolute";
     110      problems = new BindingList<ProblemInstance>();
     111      problemComboBox.DataSource = new BindingSource() { DataSource = problems };
     112      problemComboBox.DataBindings.DefaultDataSourceUpdateMode = DataSourceUpdateMode.OnPropertyChanged;
    105113      suppressUpdates = false;
    106114    }
     
    223231      }
    224232
    225       var problems = new HashSet<ProblemDescription>();
    226       foreach (var run in Content) {
    227         problems.Add(new ProblemDescription(run));
    228       }
    229 
    230       var problemTypesDifferent = problems.Select(x => x.ProblemType).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
    231       var problemNamesDifferent = problems.Select(x => x.ProblemName).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
    232       var evaluatorDifferent = problems.Select(x => x.Evaluator).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
    233       var maximizationDifferent = problems.Select(x => x.Maximization).Distinct().Count() > 1;
     233      var table = (string)dataTableComboBox.SelectedItem;
     234      var problemDict = CalculateBestTargetPerProblemInstance(table);
     235
     236      var problemTypesDifferent = problemDict.Keys.Select(x => x.ProblemType).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
     237      var problemNamesDifferent = problemDict.Keys.Select(x => x.ProblemName).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
     238      var evaluatorDifferent = problemDict.Keys.Select(x => x.Evaluator).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
     239      var maximizationDifferent = problemDict.Keys.Select(x => x.Maximization).Distinct().Count() > 1;
    234240      var allEqual = !problemTypesDifferent && !problemNamesDifferent && !evaluatorDifferent && !maximizationDifferent;
    235241
    236       var selectedProblemItem = (ProblemDescription)problemComboBox.SelectedItem;
    237       problemComboBox.Items.Clear();
    238       problemComboBox.Items.Add(ProblemDescription.MatchAll);
    239       if (selectedProblemItem == null || selectedProblemItem == ProblemDescription.MatchAll)
    240         problemComboBox.SelectedIndex = 0;
    241       foreach (var prob in problems.OrderBy(x => x.ToString()).ToList()) {
    242         prob.DisplayProblemType = problemTypesDifferent;
    243         prob.DisplayProblemName = problemNamesDifferent || allEqual;
    244         prob.DisplayEvaluator = evaluatorDifferent;
    245         prob.DisplayMaximization = maximizationDifferent;
    246         problemComboBox.Items.Add(prob);
    247         if (prob.Equals(selectedProblemItem)) problemComboBox.SelectedItem = prob;
     242      var selectedProblemItem = (ProblemInstance)problemComboBox.SelectedItem;
     243      problems.Clear();
     244      problems.Add(ProblemInstance.MatchAll);
     245      if (problems[0].Equals(selectedProblemItem)) problemComboBox.SelectedItem = problems[0];
     246      foreach (var p in problemDict.ToList()) {
     247        p.Key.BestKnownQuality = p.Value;
     248        p.Key.DisplayProblemType = problemTypesDifferent;
     249        p.Key.DisplayProblemName = problemNamesDifferent || allEqual;
     250        p.Key.DisplayEvaluator = evaluatorDifferent;
     251        p.Key.DisplayMaximization = maximizationDifferent;
     252        problems.Add(p.Key);
     253        if (p.Key.Equals(selectedProblemItem)) problemComboBox.SelectedItem = p.Key;
    248254      }
    249255      SetEnabledStateOfControls();
     
    276282    }
    277283
    278     private Dictionary<string, Dictionary<ProblemDescription, Tuple<double, List<IRun>>>> GroupRuns() {
    279       var groupedRuns = new Dictionary<string, Dictionary<ProblemDescription, Tuple<double, List<IRun>>>>();
     284    private Dictionary<string, Dictionary<ProblemInstance, List<IRun>>> GroupRuns() {
     285      var groupedRuns = new Dictionary<string, Dictionary<ProblemInstance, List<IRun>>>();
    280286
    281287      var table = (string)dataTableComboBox.SelectedItem;
     
    285291      if (string.IsNullOrEmpty(selectedGroup)) return groupedRuns;
    286292
    287       var selectedProblem = (ProblemDescription)problemComboBox.SelectedItem;
     293      var selectedProblem = (ProblemInstance)problemComboBox.SelectedItem;
    288294      if (selectedProblem == null) return groupedRuns;
    289 
    290       var targetsPerProblem = CalculateBestTargetPerProblemInstance(table);
    291 
     295     
    292296      foreach (var x in (from r in Content
    293297                         where (selectedGroup == AllRuns || r.Parameters.ContainsKey(selectedGroup))
     
    295299                           && r.Results.ContainsKey(table)
    296300                           && r.Visible
    297                          group r by selectedGroup == AllRuns ? AllRuns : r.Parameters[selectedGroup].ToString() into g
     301                         let key = selectedGroup == AllRuns ? AllRuns : r.Parameters[selectedGroup].ToString()
     302                         group r by key into g
    298303                         select Tuple.Create(g.Key, g.ToList()))) {
    299         var pDict = new Dictionary<ProblemDescription, Tuple<double, List<IRun>>>();
    300         foreach (var y in (from r in x.Item2
    301                            let pd = new ProblemDescription(r)
    302                            group r by pd into g
    303                            select Tuple.Create(g.Key, g.ToList()))) {
    304           pDict[y.Item1] = Tuple.Create(targetsPerProblem[y.Item1], y.Item2);
    305         }
    306         groupedRuns[x.Item1] = pDict;
     304        var pDict = problems.ToDictionary(r => r, _ => new List<IRun>());
     305        foreach (var y in x.Item2) {
     306          var pd = new ProblemInstance(y);
     307          List<IRun> l;
     308          if (pDict.TryGetValue(pd, out l)) l.Add(y);
     309        }
     310        groupedRuns[x.Item1] = pDict.Where(a => a.Value.Count > 0).ToDictionary(a => a.Key, a => a.Value);
    307311      }
    308312
     
    326330
    327331      var xAxisTitles = new HashSet<string>();
     332
     333      // hits describes the number of target hits at a certain time for a certain group
     334      var hits = new Dictionary<string, SortedList<double, int>>();
     335      // misses describes the number of target misses after a certain time for a certain group
     336      // for instance when a run ends, but has not achieved all targets, misses describes
     337      // how many targets have been left open at the point when the run ended
     338      var misses = new Dictionary<string, SortedList<double, int>>();
     339
     340      var aggregate = aggregateTargetsCheckBox.Checked;
     341      double minEff = double.MaxValue, maxEff = double.MinValue;
     342      var noRuns = 0;
     343      foreach (var group in groupedRuns) {
     344        SortedList<double, int> epdfHits = null, epdfMisses = null;
     345        if (aggregate) {
     346          hits[group.Key] = epdfHits = new SortedList<double, int>();
     347          misses[group.Key] = epdfMisses = new SortedList<double, int>();
     348        }
     349        foreach (var problem in group.Value) {
     350          var max = problem.Key.IsMaximization();
     351          var absTargets = GetAbsoluteTargets(problem.Key).ToArray();
     352          foreach (var run in problem.Value) {
     353            noRuns++;
     354            var resultsTable = (IndexedDataTable<double>)run.Results[table];
     355            xAxisTitles.Add(resultsTable.VisualProperties.XAxisTitle);
     356
     357            var efforts = absTargets.Select(t => GetEffortToHitTarget(resultsTable.Rows.First().Values, t, max)).ToArray();
     358            minEff = Math.Min(minEff, efforts.Min(x => x.Item2));
     359            maxEff = Math.Max(maxEff, efforts.Max(x => x.Item2));
     360            for (var idx = 0; idx < efforts.Length; idx++) {
     361              var e = efforts[idx];
     362              if (!aggregate) {
     363                var key = group.Key + "@" + (targetsAreRelative
     364                  ? (targets[idx] * 100).ToString(CultureInfo.CurrentCulture.NumberFormat) + "%"
     365                  : targets[idx].ToString(CultureInfo.CurrentCulture.NumberFormat));
     366                if (!hits.TryGetValue(key, out epdfHits))
     367                  hits[key] = epdfHits = new SortedList<double, int>();
     368                if (!misses.TryGetValue(key, out epdfMisses))
     369                  misses[key] = epdfMisses = new SortedList<double, int>();
     370              }
     371              var list = e.Item1 ? epdfHits : epdfMisses;
     372              int v;
     373              if (list.TryGetValue(e.Item2, out v))
     374                list[e.Item2] = v + 1;
     375              else list[e.Item2] = 1;
     376            }
     377          }
     378        }
     379      }
     380
     381      UpdateTargetChartAxisXBounds(minEff, maxEff);
     382
     383      DrawTargetsEcdf(hits, misses, noRuns);
     384
     385      if (targets.Length == 1) {
     386        if (targetsAreRelative)
     387          targetChart.ChartAreas[0].AxisY.Title = "Probability to be " + (targets[0] * 100) + "% worse than best";
     388        else targetChart.ChartAreas[0].AxisY.Title = "Probability to reach at least a fitness of " + targets[0];
     389      } else targetChart.ChartAreas[0].AxisY.Title = "Proportion of reached targets";
     390      targetChart.ChartAreas[0].AxisX.Title = string.Join(" / ", xAxisTitles);
     391      targetChart.ChartAreas[0].AxisX.IsLogarithmic = CanDisplayLogarithmic();
     392      targetChart.ChartAreas[0].CursorY.Interval = 0.05;
     393
     394      UpdateErtTables(groupedRuns);
     395    }
     396
     397    private void DrawTargetsEcdf(Dictionary<string, SortedList<double, int>> hits, Dictionary<string, SortedList<double, int>> misses, int noRuns) {
    328398      var colorCount = 0;
    329399      var lineStyleCount = 0;
    330 
    331       // if the group contains multiple different problem instances we want to use the
    332       // minimal maximal observed effort otherwise we run into situations where we don't
    333       // have data for a certain problem instance anymore this is a special case when
    334       // aggregating over multiple problem instances     
    335       var maxEfforts = new Dictionary<ProblemDescription, double>();
    336       double minEff = double.MaxValue, maxEff = double.MinValue;
    337       foreach (var group in groupedRuns) {
    338         foreach (var problem in group.Value) {
    339           double problemSpecificMaxEff;
    340           if (!maxEfforts.TryGetValue(problem.Key, out problemSpecificMaxEff)) {
    341             problemSpecificMaxEff = 0;
    342           }
    343           var bestKnownTarget = problem.Value.Item1;
    344           var max = problem.Key.IsMaximization();
    345           var worstTarget = (max ? (1 - targets.Max()) : (1 + targets.Max())) * bestKnownTarget;
    346           var bestTarget = (max ? (1 - targets.Min()) : (1 + targets.Min())) * bestKnownTarget;
    347           foreach (var run in problem.Value.Item2) {
    348             var row = ((IndexedDataTable<double>)run.Results[table]).Rows.First().Values;
    349             var a = row.FirstOrDefault(x => max ? x.Item2 >= worstTarget : x.Item2 <= worstTarget);
    350             var b = row.FirstOrDefault(x => max ? x.Item2 >= bestTarget : x.Item2 <= bestTarget);
    351             var firstEff = (a == default(Tuple<double, double>)) ? row.Last().Item1 : a.Item1;
    352             var lastEff = (b == default(Tuple<double, double>)) ? row.Last().Item1 : b.Item1;
    353             if (minEff > firstEff) minEff = firstEff;
    354             if (maxEff < lastEff) maxEff = lastEff;
    355             if (problemSpecificMaxEff < lastEff) problemSpecificMaxEff = lastEff;
    356           }
    357           maxEfforts[problem.Key] = problemSpecificMaxEff;
    358         }
    359       }
    360       maxEff = Math.Min(maxEff, maxEfforts.Values.Min());
    361 
     400     
     401      var showMarkers = markerCheckBox.Checked;
     402      foreach (var list in hits) {
     403        var row = new Series(list.Key) {
     404          ChartType = SeriesChartType.StepLine,
     405          BorderWidth = 3,
     406          Color = colors[colorCount],
     407          BorderDashStyle = lineStyles[lineStyleCount],
     408        };
     409        var rowShade = new Series(list.Key + "-range") {
     410          IsVisibleInLegend = false,
     411          ChartType = SeriesChartType.Range,
     412          Color = Color.FromArgb(32, colors[colorCount]),
     413          YValuesPerPoint = 2
     414        };
     415
     416        var ecdf = 0.0;
     417        var missedecdf = 0.0;
     418        var iter = misses[list.Key].GetEnumerator();
     419        var moreMisses = iter.MoveNext();
     420        var totalTargets = noRuns;
     421        if (aggregateTargetsCheckBox.Checked) totalTargets *= targets.Length;
     422        var movingTargets = totalTargets;
     423        var labelPrinted = false;
     424        foreach (var h in list.Value) {
     425          var prevmissedecdf = missedecdf;
     426          while (moreMisses && iter.Current.Key <= h.Key) {
     427            if (!labelPrinted && row.Points.Count > 0) {
     428              var point = row.Points.Last();
     429              point.Label = row.Name;
     430              point.MarkerStyle = MarkerStyle.Cross;
     431              point.MarkerBorderWidth = 1;
     432              point.MarkerSize = 10;
     433              labelPrinted = true;
     434              rowShade.Points.Add(new DataPoint(point.XValue, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
     435            }
     436            missedecdf += iter.Current.Value;
     437            movingTargets -= iter.Current.Value;
     438            if (row.Points.Count > 0 && row.Points.Last().XValue == iter.Current.Key) {
     439              row.Points.Last().SetValueY(ecdf / movingTargets);
     440              row.Points.Last().Color = Color.FromArgb(255 - (int)Math.Floor(255 * (prevmissedecdf / totalTargets)), colors[colorCount]);
     441            } else {
     442              var dp = new DataPoint(iter.Current.Key, ecdf / movingTargets) {
     443                Color = Color.FromArgb(255 - (int)Math.Floor(255 * (prevmissedecdf / totalTargets)), colors[colorCount])
     444              };
     445              if (showMarkers) {
     446                dp.MarkerStyle = MarkerStyle.Circle;
     447                dp.MarkerBorderWidth = 1;
     448                dp.MarkerSize = 5;
     449              }
     450              row.Points.Add(dp);
     451              prevmissedecdf = missedecdf;
     452            }
     453            if (boundShadingCheckBox.Checked) {
     454              if (rowShade.Points.Count > 0 && rowShade.Points.Last().XValue == iter.Current.Key)
     455                rowShade.Points.Last().SetValueY(ecdf / totalTargets, (ecdf + missedecdf) / totalTargets);
     456              else rowShade.Points.Add(new DataPoint(iter.Current.Key, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
     457            }
     458            moreMisses = iter.MoveNext();
     459            if (!labelPrinted) {
     460              var point = row.Points.Last();
     461              point.Label = row.Name;
     462              point.MarkerStyle = MarkerStyle.Cross;
     463              point.MarkerBorderWidth = 1;
     464              point.MarkerSize = 10;
     465              labelPrinted = true;
     466            }
     467          }
     468          ecdf += h.Value;
     469          if (row.Points.Count > 0 && row.Points.Last().XValue == h.Key) {
     470            row.Points.Last().SetValueY(ecdf / movingTargets);
     471            row.Points.Last().Color = Color.FromArgb(255 - (int)Math.Floor(255 * (missedecdf / totalTargets)), colors[colorCount]);
     472          } else {
     473            var dp = new DataPoint(h.Key, ecdf / movingTargets) {
     474              Color = Color.FromArgb(255 - (int)Math.Floor(255 * (missedecdf / totalTargets)), colors[colorCount])
     475            };
     476            if (showMarkers) {
     477              dp.MarkerStyle = MarkerStyle.Circle;
     478              dp.MarkerBorderWidth = 1;
     479              dp.MarkerSize = 5;
     480            }
     481            row.Points.Add(dp);
     482          }
     483          if (missedecdf > 0 && boundShadingCheckBox.Checked) {
     484            if (rowShade.Points.Count > 0 && rowShade.Points.Last().XValue == h.Key)
     485              rowShade.Points.Last().SetValueY(ecdf / totalTargets, (ecdf + missedecdf) / totalTargets);
     486            else rowShade.Points.Add(new DataPoint(h.Key, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
     487          }
     488        }
     489
     490        while (moreMisses) {
     491          // if there are misses beyond the last hit we extend the shaded area
     492          missedecdf += iter.Current.Value;
     493          var dp = new DataPoint(iter.Current.Key, ecdf / movingTargets) {
     494            Color = Color.FromArgb(255 - (int)Math.Floor(255 * (missedecdf / totalTargets)), colors[colorCount])
     495          };
     496          if (showMarkers) {
     497            dp.MarkerStyle = MarkerStyle.Circle;
     498            dp.MarkerBorderWidth = 1;
     499            dp.MarkerSize = 5;
     500          }
     501          row.Points.Add(dp);
     502          if (boundShadingCheckBox.Checked) {
     503            rowShade.Points.Add(new DataPoint(iter.Current.Key, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
     504          }
     505          moreMisses = iter.MoveNext();
     506
     507          if (!labelPrinted && row.Points.Count > 0) {
     508            var point = row.Points.Last();
     509            point.Label = row.Name;
     510            point.MarkerStyle = MarkerStyle.Cross;
     511            point.MarkerBorderWidth = 1;
     512            point.MarkerSize = 10;
     513            labelPrinted = true;
     514          }
     515        }
     516
     517        if (!labelPrinted) {
     518          var point = row.Points.Last();
     519          point.Label = row.Name;
     520          point.MarkerStyle = MarkerStyle.Cross;
     521          point.MarkerBorderWidth = 1;
     522          point.MarkerSize = 10;
     523          rowShade.Points.Add(new DataPoint(point.XValue, new[] {ecdf / totalTargets, (ecdf + missedecdf) / totalTargets}));
     524          labelPrinted = true;
     525        }
     526
     527        ConfigureSeries(row);
     528        targetChart.Series.Add(rowShade);
     529        targetChart.Series.Add(row);
     530
     531        colorCount = (colorCount + 1) % colors.Length;
     532        if (colorCount == 0) lineStyleCount = (lineStyleCount + 1) % lineStyles.Length;
     533      }
     534    }
     535
     536    private void UpdateTargetChartAxisXBounds(double minEff, double maxEff) {
    362537      var minZeros = (int)Math.Floor(Math.Log10(minEff));
    363538      var maxZeros = (int)Math.Floor(Math.Log10(maxEff));
     
    372547      targetChart.ChartAreas[0].AxisX.Minimum = (double)axisMin;
    373548      targetChart.ChartAreas[0].AxisX.Maximum = (double)axisMax;
    374 
    375       foreach (var group in groupedRuns) {
    376         // hits describes the number of target hits at a certain time for a certain group
    377         var hits = new Dictionary<string, SortedList<double, int>>();
    378         // misses describes the number of target misses after a certain time for a certain group
    379         // for instance when a run ends, but has not achieved all targets, misses describes
    380         // how many targets have been left open at the point when the run ended
    381         var misses = new Dictionary<string, SortedList<double, int>>();
    382         var maxLength = 0.0;
    383 
    384         var noRuns = 0;
    385         foreach (var problem in group.Value) {
    386           foreach (var run in problem.Value.Item2) {
    387             var resultsTable = (IndexedDataTable<double>)run.Results[table];
    388             xAxisTitles.Add(resultsTable.VisualProperties.XAxisTitle);
    389 
    390             if (aggregateTargetsCheckBox.Checked) {
    391               var length = CalculateHitsForAllTargets(hits, misses, resultsTable.Rows.First(), problem.Key, group.Key, problem.Value.Item1, maxEff);
    392               maxLength = Math.Max(length, maxLength);
    393             } else {
    394               CalculateHitsForEachTarget(hits, misses, resultsTable.Rows.First(), problem.Key, group.Key, problem.Value.Item1, maxEff);
    395             }
    396             noRuns++;
    397           }
    398         }
    399         foreach (var list in hits) {
    400           var row = new Series(list.Key) {
    401             ChartType = SeriesChartType.StepLine,
    402             BorderWidth = 2,
    403             Color = colors[colorCount],
    404             BorderDashStyle = lineStyles[lineStyleCount],
    405           };
    406           var rowShade = new Series(list.Key + "-range") {
    407             IsVisibleInLegend = false,
    408             ChartType = SeriesChartType.Range,
    409             Color = Color.FromArgb(32, colors[colorCount])
    410           };
    411 
    412           var ecdf = 0.0;
    413           var missedecdf = 0.0;
    414           var iter = misses[list.Key].GetEnumerator();
    415           var moreMisses = iter.MoveNext();
    416           var totalTargets = noRuns;
    417           if (aggregateTargetsCheckBox.Checked) totalTargets *= targets.Length;
    418           var movingTargets = totalTargets;
    419           foreach (var h in list.Value) {
    420             while (moreMisses && iter.Current.Key <= h.Key) {
    421               missedecdf += iter.Current.Value;
    422               movingTargets -= iter.Current.Value;
    423               if (row.Points.Count > 0 && row.Points.Last().XValue == iter.Current.Key)
    424                 row.Points.Last().SetValueY(ecdf / movingTargets);
    425               else row.Points.AddXY(iter.Current.Key, ecdf / movingTargets);
    426               if (boundShadingCheckBox.Checked) {
    427                 if (rowShade.Points.Count > 0 && rowShade.Points.Last().XValue == iter.Current.Key)
    428                   rowShade.Points.Last().SetValueY(ecdf / totalTargets, (ecdf + missedecdf) / totalTargets);
    429                 else rowShade.Points.Add(new DataPoint(iter.Current.Key, new[] { ecdf / totalTargets, (ecdf + missedecdf) / totalTargets }));
    430               }
    431               moreMisses = iter.MoveNext();
    432             }
    433             ecdf += h.Value;
    434             if (row.Points.Count > 0 && row.Points.Last().XValue == h.Key)
    435               row.Points.Last().SetValueY(ecdf / movingTargets);
    436             else row.Points.AddXY(h.Key, ecdf / movingTargets);
    437             if (missedecdf > 0 && boundShadingCheckBox.Checked) {
    438               if (rowShade.Points.Count > 0 && rowShade.Points.Last().XValue == h.Key)
    439                 rowShade.Points.Last().SetValueY(ecdf / totalTargets, (ecdf + missedecdf) / totalTargets);
    440               else rowShade.Points.Add(new DataPoint(h.Key, new[] { ecdf / totalTargets, (ecdf + missedecdf) / totalTargets }));
    441             }
    442           }
    443 
    444           while (moreMisses) {
    445             // if there are misses beyond the last hit we extend the shaded area
    446             missedecdf += iter.Current.Value;
    447             //movingTargets -= iter.Current.Value;
    448             if (row.Points.Count > 0 && row.Points.Last().XValue == iter.Current.Key)
    449               row.Points.Last().SetValueY(ecdf / movingTargets);
    450             else row.Points.AddXY(iter.Current.Key, ecdf / movingTargets);
    451             if (boundShadingCheckBox.Checked) {
    452               if (rowShade.Points.Count > 0 && rowShade.Points.Last().XValue == iter.Current.Key)
    453                 rowShade.Points.Last().SetValueY(ecdf / totalTargets, (ecdf + missedecdf) / totalTargets);
    454               else rowShade.Points.Add(new DataPoint(iter.Current.Key, new[] { ecdf / totalTargets, (ecdf + missedecdf) / totalTargets }));
    455             }
    456             moreMisses = iter.MoveNext();
    457           }
    458 
    459           if (maxLength > 0 && (row.Points.Count == 0 || row.Points.Last().XValue < maxLength))
    460             row.Points.AddXY(maxLength, ecdf / movingTargets);
    461 
    462           if (row.Points.Count > 0) {
    463             var point = row.Points.Last();
    464             point.Label = row.Name;
    465             point.MarkerStyle = MarkerStyle.Cross;
    466             point.MarkerBorderWidth = 1;
    467           }
    468 
    469           ConfigureSeries(row);
    470           targetChart.Series.Add(rowShade);
    471           targetChart.Series.Add(row);
    472         }
    473         colorCount = (colorCount + 1) % colors.Length;
    474         if (colorCount == 0) lineStyleCount = (lineStyleCount + 1) % lineStyles.Length;
    475       }
    476 
    477       if (targets.Length == 1)
    478         targetChart.ChartAreas[0].AxisY.Title = "Probability to be " + (targets[0] * 100) + "% worse than best";
    479       else targetChart.ChartAreas[0].AxisY.Title = "Proportion of reached targets";
    480       targetChart.ChartAreas[0].AxisX.Title = string.Join(" / ", xAxisTitles);
    481       targetChart.ChartAreas[0].AxisX.IsLogarithmic = CanDisplayLogarithmic();
    482       targetChart.ChartAreas[0].CursorY.Interval = 0.05;
    483       UpdateErtTables(groupedRuns);
     549    }
     550
     551    private IEnumerable<double> GetAbsoluteTargets(ProblemInstance pInstance) {
     552      if (!targetsAreRelative) return targets;
     553      var maximization = pInstance.IsMaximization();
     554      var bestKnown = pInstance.BestKnownQuality;
     555      if (double.IsNaN(bestKnown)) throw new ArgumentException("Problem instance does not have a defined best - known quality.");
     556      IEnumerable<double> tmp = null;
     557      if (bestKnown > 0) {
     558        tmp = targets.Select(x => (maximization ? (1 - x) : (1 + x)) * bestKnown);
     559      } else if (bestKnown < 0) {
     560        tmp = targets.Select(x => (!maximization ? (1 - x) : (1 + x)) * bestKnown);
     561      } else {
     562        // relative to 0 is impossible
     563        tmp = targets;
     564      }
     565      return tmp;
     566    }
     567
     568    private double[] GetAbsoluteTargetsWorstToBest(ProblemInstance pInstance) {
     569      if (double.IsNaN(pInstance.BestKnownQuality)) throw new ArgumentException("Problem instance does not have a defined best-known quality.");
     570      var absTargets = GetAbsoluteTargets(pInstance);
     571      return (pInstance.IsMaximization()
     572        ? absTargets.OrderBy(x => x) : absTargets.OrderByDescending(x => x)).ToArray();
    484573    }
    485574
    486575    private void GenerateDefaultTargets() {
    487       targets = new[] { 0.1, 0.05, 0.02, 0.01, 0 };
     576      targets = new[] { 0.1, 0.095, 0.09, 0.085, 0.08, 0.075, 0.07, 0.065, 0.06, 0.055, 0.05, 0.045, 0.04, 0.035, 0.03, 0.025, 0.02, 0.015, 0.01, 0.005, 0 };
    488577      suppressTargetsEvents = true;
    489578      targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100)) + "%";
     
    491580    }
    492581
    493     private void CalculateHitsForEachTarget(Dictionary<string, SortedList<double, int>> hits,
    494                                             Dictionary<string, SortedList<double, int>> misses,
    495                                             IndexedDataRow<double> row, ProblemDescription problem,
    496                                             string group, double bestTarget, double maxEffort) {
    497       foreach (var t in targets.Select(x => Tuple.Create((problem.IsMaximization() ? (1 - x) : (1 + x)) * bestTarget, x))) {
    498         var l = t.Item1;
    499         var key = group + "_" + (t.Item2 * 100) + "%_" + l;
    500         if (!hits.ContainsKey(key)) {
    501           hits.Add(key, new SortedList<double, int>());
    502           misses.Add(key, new SortedList<double, int>());
    503         }
    504         var hit = false;
    505         foreach (var v in row.Values) {
    506           if (v.Item1 > maxEffort) break;
    507           if (problem.IsMaximization() && v.Item2 >= l || !problem.IsMaximization() && v.Item2 <= l) {
    508             if (hits[key].ContainsKey(v.Item1))
    509               hits[key][v.Item1]++;
    510             else hits[key][v.Item1] = 1;
    511             hit = true;
    512             break;
    513           }
    514         }
    515         if (!hit) {
    516           var max = Math.Min(row.Values.Last().Item1, maxEffort);
    517           if (misses[key].ContainsKey(max))
    518             misses[key][max]++;
    519           else misses[key][max] = 1;
    520         }
    521       }
    522     }
    523 
    524     private double CalculateHitsForAllTargets(Dictionary<string, SortedList<double, int>> hits,
    525                                               Dictionary<string, SortedList<double, int>> misses,
    526                                               IndexedDataRow<double> row, ProblemDescription problem,
    527                                               string group, double bestTarget, double maxEffort) {
    528       var values = row.Values;
    529       if (!hits.ContainsKey(group)) {
    530         hits.Add(group, new SortedList<double, int>());
    531         misses.Add(group, new SortedList<double, int>());
    532       }
    533 
    534       var i = 0;
    535       var j = 0;
    536       while (i < targets.Length && j < values.Count) {
    537         var target = (problem.IsMaximization() ? (1 - targets[i]) : (1 + targets[i])) * bestTarget;
    538         var current = values[j];
    539         if (current.Item1 > maxEffort) break;
    540         if (problem.IsMaximization() && current.Item2 >= target
    541           || !problem.IsMaximization() && current.Item2 <= target) {
    542           if (hits[group].ContainsKey(current.Item1)) hits[group][current.Item1]++;
    543           else hits[group][current.Item1] = 1;
    544           i++;
    545         } else {
    546           j++;
    547         }
    548       }
    549       if (j == values.Count) j--;
    550       var effort = Math.Min(values[j].Item1, maxEffort);
    551       if (i < targets.Length) {
    552         if (misses[group].ContainsKey(effort))
    553           misses[group][effort] += targets.Length - i;
    554         else misses[group][effort] = targets.Length - i;
    555       }
    556       return effort;
    557     }
    558 
    559     private void UpdateErtTables(Dictionary<string, Dictionary<ProblemDescription, Tuple<double, List<IRun>>>> groupedRuns) {
     582    private Tuple<bool, double> GetEffortToHitTarget(
     583        IEnumerable<Tuple<double, double>> convergenceGraph,
     584        double absTarget, bool maximization) {
     585      var hit = false;
     586      var effort = double.NaN;
     587      foreach (var dent in convergenceGraph) {
     588        effort = dent.Item1;
     589        hit = maximization && dent.Item2 >= absTarget || !maximization && dent.Item2 <= absTarget;
     590        if (hit) break;
     591      }
     592      if (double.IsNaN(effort)) throw new ArgumentException("Convergence graph is empty.", "convergenceGraph");
     593      return Tuple.Create(hit, effort);
     594    }
     595
     596    private void UpdateErtTables(Dictionary<string, Dictionary<ProblemInstance, List<IRun>>> groupedRuns) {
    560597      ertTableView.Content = null;
    561598      var columns = 1 + targets.Length + 1;
     
    565602      var tableName = (string)dataTableComboBox.SelectedItem;
    566603      if (string.IsNullOrEmpty(tableName)) return;
    567 
    568       var targetsPerProblem = CalculateBestTargetPerProblemInstance(tableName);
    569 
    570       var colNames = new string[columns];
    571       colNames[0] = colNames[colNames.Length - 1] = string.Empty;
    572       for (var i = 0; i < targets.Length; i++) {
    573         colNames[i + 1] = targets[i].ToString("0.0%");
    574       }
     604     
    575605      var problems = groupedRuns.SelectMany(x => x.Value.Keys).Distinct().ToList();
    576606
    577607      foreach (var problem in problems) {
     608        var max = problem.IsMaximization();
    578609        matrix[rowCount, 0] = problem.ToString();
    579         for (var i = 0; i < targets.Length; i++) {
    580           matrix[rowCount, i + 1] = (targetsPerProblem[problem] * (problem.IsMaximization() ? (1 - targets[i]) : (1 + targets[i]))).ToString(CultureInfo.CurrentCulture.NumberFormat);
     610        var absTargets = GetAbsoluteTargetsWorstToBest(problem);
     611        for (var i = 0; i < absTargets.Length; i++) {
     612          matrix[rowCount, i + 1] = absTargets[i].ToString(CultureInfo.CurrentCulture.NumberFormat);
    581613        }
    582614        matrix[rowCount, columns - 1] = "#succ";
     
    590622            continue;
    591623          }
    592           var runs = group.Value[problem].Item2;
    593           ErtCalculationResult result = default(ErtCalculationResult);
    594           for (var i = 0; i < targets.Length; i++) {
    595             result = ExpectedRuntimeHelper.CalculateErt(runs, tableName, (problem.IsMaximization() ? (1 - targets[i]) : (1 + targets[i])) * group.Value[problem].Item1, problem.IsMaximization());
     624          var runs = group.Value[problem];
     625          var result = default(ErtCalculationResult);
     626          for (var i = 0; i < absTargets.Length; i++) {
     627            result = ExpectedRuntimeHelper.CalculateErt(runs, tableName, absTargets[i], max);
    596628            matrix[rowCount, i + 1] = result.ToString();
    597629          }
     
    600632        }
    601633      }
    602       ertTableView.Content = new StringMatrix(matrix) { ColumnNames = colNames };
     634      ertTableView.Content = new StringMatrix(matrix);
    603635      ertTableView.DataGridView.AutoResizeColumns(DataGridViewAutoSizeColumnsMode.AllCells);
    604636    }
     
    621653      var colorCount = 0;
    622654      var lineStyleCount = 0;
    623 
    624       var targetsPerProblem = CalculateBestTargetPerProblemInstance((string)dataTableComboBox.SelectedItem);
    625 
     655     
    626656      foreach (var group in groupedRuns) {
    627657        var hits = new Dictionary<string, SortedList<double, double>>();
    628658
    629659        foreach (var problem in group.Value) {
    630           foreach (var run in problem.Value.Item2) {
     660          foreach (var run in problem.Value) {
    631661            var resultsTable = (IndexedDataTable<double>)run.Results[table];
    632662
    633             if (eachOrAllBudgetsCheckBox.Checked) {
    634               CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), group.Value.Count, problem.Key, group.Key, problem.Value.Item2.Count, targetsPerProblem[problem.Key]);
     663            if (aggregateBudgetsCheckBox.Checked) {
     664              CalculateHitsForAllBudgets(hits, resultsTable.Rows.First(), group.Value.Count, problem.Key, group.Key, problem.Value.Count);
    635665            } else {
    636               CalculateHitsForAllBudgets(hits, resultsTable.Rows.First(), group.Value.Count, problem.Key, group.Key, problem.Value.Item2.Count, targetsPerProblem[problem.Key]);
     666              CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), group.Value.Count, problem.Key, group.Key, problem.Value.Count);
    637667            }
    638668          }
     
    667697
    668698    private void GenerateDefaultBudgets(string table) {
    669       var runs = GroupRuns().SelectMany(x => x.Value.Values).SelectMany(x => x.Item2).ToList();
     699      var runs = GroupRuns().SelectMany(x => x.Value.Values).SelectMany(x => x).ToList();
    670700      var min = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Min()).Min();
    671701      var max = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Max()).Max();
    672 
    673       var maxMagnitude = (int)Math.Ceiling(Math.Log10(max));
    674       var minMagnitude = (int)Math.Floor(Math.Log10(min));
    675       if (maxMagnitude - minMagnitude >= 3) {
    676         budgets = new double[maxMagnitude - minMagnitude];
    677         for (var i = minMagnitude; i < maxMagnitude; i++) {
    678           budgets[i - minMagnitude] = Math.Pow(10, i);
    679         }
    680       } else {
    681         var range = max - min;
    682         budgets = Enumerable.Range(0, 6).Select(x => min + (x / 5.0) * range).ToArray();
    683       }
     702      var points = 20;
     703      budgets = Enumerable.Range(1, points).Select(x => min + (x / (double)points) * (max - min)).ToArray();
    684704      suppressBudgetsEvents = true;
    685705      budgetsTextBox.Text = string.Join(" ; ", budgets);
     
    687707    }
    688708
    689     private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, ProblemDescription problem, string groupName, int problemCount, double bestTarget) {
     709    private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, ProblemInstance problem, string groupName, int problemCount) {
     710      var max = problem.IsMaximization();
    690711      foreach (var b in budgets) {
    691712        var key = groupName + "-" + b;
     
    697718            if (prev == null && v.Item1 != b) break;
    698719            var tgt = ((prev == null || v.Item1 == b) ? v.Item2 : prev.Item2);
    699             tgt = problem.IsMaximization() ? bestTarget / tgt : tgt / bestTarget;
    700             if (hits[key].ContainsKey(tgt))
    701               hits[key][tgt] += 1.0 / (groupCount * problemCount);
    702             else hits[key][tgt] = 1.0 / (groupCount * problemCount);
     720            var relTgt = CalculateRelativeDifference(max, problem.BestKnownQuality, tgt) + 1;
     721            if (hits[key].ContainsKey(relTgt))
     722              hits[key][relTgt] += 1.0 / (groupCount * problemCount);
     723            else hits[key][relTgt] = 1.0 / (groupCount * problemCount);
    703724            break;
    704725          }
     
    709730    }
    710731
    711     private void CalculateHitsForAllBudgets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, ProblemDescription problem, string groupName, int problemCount, double bestTarget) {
     732    private void CalculateHitsForAllBudgets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, ProblemInstance problem, string groupName, int problemCount) {
    712733      var values = row.Values;
    713734      if (!hits.ContainsKey(groupName)) hits.Add(groupName, new SortedList<double, double>());
     
    716737      var j = 0;
    717738      Tuple<double, double> prev = null;
     739      var max = problem.IsMaximization();
    718740      while (i < budgets.Length && j < values.Count) {
    719741        var current = values[j];
     
    721743          if (prev != null || current.Item1 == budgets[i]) {
    722744            var tgt = (prev == null || current.Item1 == budgets[i]) ? current.Item2 : prev.Item2;
    723             tgt = problem.IsMaximization() ? bestTarget / tgt : tgt / bestTarget;
    724             if (!hits[groupName].ContainsKey(tgt)) hits[groupName][tgt] = 0;
    725             hits[groupName][tgt] += 1.0 / (groupCount * problemCount * budgets.Length);
     745            var relTgt = CalculateRelativeDifference(max, problem.BestKnownQuality, tgt) + 1;
     746            if (!hits[groupName].ContainsKey(relTgt)) hits[groupName][relTgt] = 0;
     747            hits[groupName][relTgt] += 1.0 / (groupCount * problemCount * budgets.Length);
    726748          }
    727749          i++;
     
    732754      }
    733755      var lastTgt = values.Last().Item2;
    734       lastTgt = problem.IsMaximization() ? bestTarget / lastTgt : lastTgt / bestTarget;
    735       if (i < budgets.Length && !hits[groupName].ContainsKey(lastTgt)) hits[groupName][lastTgt] = 0;
     756      var lastRelTgt = CalculateRelativeDifference(max, problem.BestKnownQuality, lastTgt) + 1;
     757      if (i < budgets.Length && !hits[groupName].ContainsKey(lastRelTgt)) hits[groupName][lastRelTgt] = 0;
    736758      while (i < budgets.Length) {
    737         hits[groupName][lastTgt] += 1.0 / (groupCount * problemCount * budgets.Length);
     759        hits[groupName][lastRelTgt] += 1.0 / (groupCount * problemCount * budgets.Length);
    738760        i++;
    739761      }
     
    756778      if (dataTableComboBox.SelectedIndex >= 0)
    757779        GenerateDefaultBudgets((string)dataTableComboBox.SelectedItem);
     780      UpdateBestKnownQualities();
    758781      UpdateRuns();
    759782      SetEnabledStateOfControls();
     
    782805          return;
    783806        }
    784         targetList.Add(t / 100);
     807        if (targetsAreRelative)
     808          targetList.Add(t / 100);
     809        else targetList.Add(t);
    785810      }
    786811      if (targetList.Count == 0) {
     
    791816      e.Cancel = false;
    792817      errorProvider.SetError(targetsTextBox, null);
    793       targets = targetList.Select(x => (double)x).OrderByDescending(x => x).ToArray();
     818      targets = targetsAreRelative ? targetList.Select(x => (double)x).OrderByDescending(x => x).ToArray() : targetList.Select(x => (double)x).ToArray();
    794819      suppressTargetsEvents = true;
    795       try { targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100)) + "%"; } finally { suppressTargetsEvents = false; }
     820      try {
     821        if (targetsAreRelative)
     822          targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100)) + "%";
     823        else targetsTextBox.Text = string.Join(" ; ", targets);
     824      } finally { suppressTargetsEvents = false; }
    796825
    797826      UpdateResultsByTarget();
     
    804833        UpdateResultsByTarget();
    805834      } finally { ResumeRepaint(true); }
     835    }
     836
     837    private void relativeOrAbsoluteComboBox_SelectedIndexChanged(object sender, EventArgs e) {
     838      if (suppressUpdates) return;
     839      var pd = (ProblemInstance)problemComboBox.SelectedItem;
     840      if (!double.IsNaN(pd.BestKnownQuality)) {
     841        var max = pd.IsMaximization();
     842        if (targetsAreRelative) targets = GetAbsoluteTargets(pd).ToArray();
     843        else {
     844          // Rounding to 5 digits since it's certainly appropriate for this application
     845          if (pd.BestKnownQuality > 0) {
     846            targets = targets.Select(x => Math.Round(max ? 1.0 - (x / pd.BestKnownQuality) : (x / pd.BestKnownQuality) - 1.0, 5)).ToArray();
     847          } else if (pd.BestKnownQuality < 0) {
     848            targets = targets.Select(x => Math.Round(!max ? 1.0 - (x / pd.BestKnownQuality) : (x / pd.BestKnownQuality) - 1.0, 5)).ToArray();
     849          }
     850        }
     851      }
     852      targetsAreRelative = (string)relativeOrAbsoluteComboBox.SelectedItem == "relative";
     853      SuspendRepaint();
     854      suppressTargetsEvents = true;
     855      try {
     856        if (targetsAreRelative)
     857          targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100)) + "%";
     858        else targetsTextBox.Text = string.Join(" ; ", targets);
     859
     860        UpdateResultsByTarget();
     861      } finally { suppressTargetsEvents = false; ResumeRepaint(true); }
    806862    }
    807863
     
    823879            targets = dialog.Values.Select(x => (double)x).ToArray();
    824880            suppressTargetsEvents = true;
    825             targetsTextBox.Text = string.Join("% ; ", targets);
     881            if (targetsAreRelative)
     882              targetsTextBox.Text = string.Join("% ; ", targets);
     883            else targetsTextBox.Text = string.Join(" ; ", targets);
    826884            suppressTargetsEvents = false;
    827885
     
    835893    private void addTargetsAsResultButton_Click(object sender, EventArgs e) {
    836894      var table = (string)dataTableComboBox.SelectedItem;
    837 
    838       var targetsPerProblem = CalculateBestTargetPerProblemInstance(table);
    839 
     895      if (string.IsNullOrEmpty(table)) return;
     896     
    840897      foreach (var run in Content) {
    841898        if (!run.Results.ContainsKey(table)) continue;
     
    844901        var i = 0;
    845902        var j = 0;
    846         var pd = new ProblemDescription(run);
    847         while (i < targets.Length && j < values.Count) {
    848           var target = (pd.IsMaximization() ? (1 - targets[i]) : (1 + targets[i])) * targetsPerProblem[pd];
     903        var pd = new ProblemInstance(run);
     904        var max = pd.IsMaximization();
     905        var absTargets = GetAbsoluteTargetsWorstToBest(problems.Single(x => x.Equals(pd)));
     906        while (i < absTargets.Length && j < values.Count) {
     907          var target = absTargets[i];
    849908          var current = values[j];
    850           if (pd.IsMaximization() && current.Item2 >= target
    851               || !pd.IsMaximization() && current.Item2 <= target) {
     909          if (max && current.Item2 >= target || !max && current.Item2 <= target) {
    852910            run.Results[table + ".Target" + target] = new DoubleValue(current.Item1);
    853911            i++;
     
    857915        }
    858916      }
     917    }
     918
     919    private void markerCheckBox_CheckedChanged(object sender, EventArgs e) {
     920      SuspendRepaint();
     921      try {
     922        UpdateResultsByTarget();
     923      } finally { ResumeRepaint(true); }
    859924    }
    860925    #endregion
     
    887952    }
    888953
    889     private void eachOrAllBudgetsCheckBox_CheckedChanged(object sender, EventArgs e) {
    890       var each = eachOrAllBudgetsCheckBox.Checked;
    891       eachOrAllBudgetsCheckBox.Text = each ? "each" : "all";
     954    private void aggregateBudgetsCheckBox_CheckedChanged(object sender, EventArgs e) {
    892955      SuspendRepaint();
    893956      try {
     
    9671030
    9681031    #region Helpers
    969     private Dictionary<ProblemDescription, double> CalculateBestTargetPerProblemInstance(string table) {
     1032    private double CalculateRelativeDifference(bool maximization, double bestKnown, double fit) {
     1033      if (bestKnown == 0) {
     1034        // no relative difference with respect to bestKnown possible
     1035        return maximization ? -fit : fit;
     1036      }
     1037      var absDiff = (fit - bestKnown);
     1038      var relDiff = absDiff / bestKnown;
     1039      if (maximization) {
     1040        return bestKnown > 0 ? -relDiff : relDiff;
     1041      } else {
     1042        return bestKnown > 0 ? relDiff : -relDiff;
     1043      }
     1044    }
     1045
     1046    private Dictionary<ProblemInstance, double> CalculateBestTargetPerProblemInstance(string table) {
     1047      if (table == null) table = string.Empty;
    9701048      return (from r in Content
    9711049              where r.Visible
    972               let pd = new ProblemDescription(r)
     1050              let pd = new ProblemInstance(r)
    9731051              let target = r.Parameters.ContainsKey("BestKnownQuality")
    9741052                           && r.Parameters["BestKnownQuality"] is DoubleValue
    9751053                ? ((DoubleValue)r.Parameters["BestKnownQuality"]).Value
    976                 : ((IndexedDataTable<double>)r.Results[table]).Rows.First().Values.Last().Item2
     1054                : (r.Results.ContainsKey(table) && r.Results[table] is IndexedDataTable<double>
     1055                  ? ((IndexedDataTable<double>)r.Results[table]).Rows.First().Values.Last().Item2
     1056                  : double.NaN)
    9771057              group target by pd into g
    978               select new { Problem = g.Key, Target = g.Key.IsMaximization() ? g.Max() : g.Min() })
     1058              select new { Problem = g.Key, Target = g.Any(x => !double.IsNaN(x)) ? (g.Key.IsMaximization() ? g.Max() : g.Where(x => !double.IsNaN(x)).Min()) : double.NaN })
    9791059        .ToDictionary(x => x.Problem, x => x.Target);
    9801060    }
     
    9901070        UpdateResultsByCost();
    9911071      } finally { ResumeRepaint(true); }
     1072    }
     1073
     1074    private void UpdateBestKnownQualities() {
     1075      var table = (string)dataTableComboBox.SelectedItem;
     1076      if (string.IsNullOrEmpty(table)) return;
     1077
     1078      var targetsPerProblem = CalculateBestTargetPerProblemInstance(table);
     1079      foreach (var pd in problems) {
     1080        double bkq;
     1081        if (targetsPerProblem.TryGetValue(pd, out bkq))
     1082          pd.BestKnownQuality = bkq;
     1083        else pd.BestKnownQuality = double.NaN;
     1084      }
     1085      //problemComboBox.ResetBindings();
    9921086    }
    9931087    #endregion
     
    10781172    }
    10791173
    1080     private class ProblemDescription {
     1174    private class ProblemInstance : INotifyPropertyChanged {
    10811175      private readonly bool matchAll;
    1082       public static readonly ProblemDescription MatchAll = new ProblemDescription() {
     1176      public static readonly ProblemInstance MatchAll = new ProblemInstance() {
    10831177        ProblemName = "All with Best-Known"
    10841178      };
    10851179
    1086       private ProblemDescription() {
     1180      private ProblemInstance() {
    10871181        ProblemType = string.Empty;
    10881182        ProblemName = string.Empty;
     
    10941188        DisplayMaximization = false;
    10951189        matchAll = true;
    1096       }
    1097 
    1098       public ProblemDescription(IRun run) {
     1190        BestKnownQuality = double.NaN;
     1191      }
     1192
     1193      public ProblemInstance(IRun run) {
    10991194        ProblemType = GetStringValueOrEmpty(run, "Problem Type");
    11001195        ProblemName = GetStringValueOrEmpty(run, "Problem Name");
     
    11061201        DisplayMaximization = !string.IsNullOrEmpty(Maximization);
    11071202        matchAll = false;
    1108       }
    1109 
    1110       public bool DisplayProblemType { get; set; }
    1111       public string ProblemType { get; set; }
    1112       public bool DisplayProblemName { get; set; }
    1113       public string ProblemName { get; set; }
    1114       public bool DisplayEvaluator { get; set; }
    1115       public string Evaluator { get; set; }
    1116       public bool DisplayMaximization { get; set; }
    1117       public string Maximization { get; set; }
     1203        BestKnownQuality = GetDoubleValueOrNaN(run, "BestKnownQuality");
     1204      }
     1205
     1206      private bool displayProblemType;
     1207      public bool DisplayProblemType {
     1208        get { return displayProblemType; }
     1209        set {
     1210          if (displayProblemType == value) return;
     1211          displayProblemType = value;
     1212          OnPropertyChanged("DisplayProblemType");
     1213        }
     1214      }
     1215      private string problemType;
     1216      public string ProblemType {
     1217        get { return problemType; }
     1218        set {
     1219          if (problemType == value) return;
     1220          problemType = value;
     1221          OnPropertyChanged("ProblemType");
     1222        }
     1223      }
     1224      private bool displayProblemName;
     1225      public bool DisplayProblemName {
     1226        get { return displayProblemName; }
     1227        set {
     1228          if (displayProblemName == value) return;
     1229          displayProblemName = value;
     1230          OnPropertyChanged("DisplayProblemName");
     1231        }
     1232      }
     1233      private string problemName;
     1234      public string ProblemName {
     1235        get { return problemName; }
     1236        set {
     1237          if (problemName == value) return;
     1238          problemName = value;
     1239          OnPropertyChanged("ProblemName");
     1240        }
     1241      }
     1242      private bool displayEvaluator;
     1243      public bool DisplayEvaluator {
     1244        get { return displayEvaluator; }
     1245        set {
     1246          if (displayEvaluator == value) return;
     1247          displayEvaluator = value;
     1248          OnPropertyChanged("DisplayEvaluator");
     1249        }
     1250      }
     1251      private string evaluator;
     1252      public string Evaluator {
     1253        get { return evaluator; }
     1254        set {
     1255          if (evaluator == value) return;
     1256          evaluator = value;
     1257          OnPropertyChanged("Evaluator");
     1258        }
     1259      }
     1260      private bool displayMaximization;
     1261      public bool DisplayMaximization {
     1262        get { return displayMaximization; }
     1263        set {
     1264          if (displayMaximization == value) return;
     1265          displayMaximization = value;
     1266          OnPropertyChanged("DisplayMaximization");
     1267        }
     1268      }
     1269      private string maximization;
     1270      public string Maximization {
     1271        get { return maximization; }
     1272        set {
     1273          if (maximization == value) return;
     1274          maximization = value;
     1275          OnPropertyChanged("Maximization");
     1276        }
     1277      }
     1278      private double bestKnownQuality;
     1279      public double BestKnownQuality {
     1280        get { return bestKnownQuality; }
     1281        set {
     1282          if (bestKnownQuality == value) return;
     1283          bestKnownQuality = value;
     1284          OnPropertyChanged("BestKnownQuality");
     1285        }
     1286      }
    11181287
    11191288      public bool IsMaximization() {
     
    11291298      }
    11301299
     1300      private double GetDoubleValueOrNaN(IRun run, string key) {
     1301        IItem param;
     1302        if (run.Parameters.TryGetValue(key, out param)) {
     1303          var dv = param as DoubleValue;
     1304          return dv != null ? dv.Value : double.NaN;
     1305        }
     1306        return double.NaN;
     1307      }
     1308
    11311309      private string GetStringValueOrEmpty(IRun run, string key) {
    1132         return run.Parameters.ContainsKey(key) ? ((StringValue)run.Parameters[key]).Value : string.Empty;
     1310        IItem param;
     1311        if (run.Parameters.TryGetValue(key, out param)) {
     1312          var sv = param as StringValue;
     1313          return sv != null ? sv.Value : string.Empty;
     1314        }
     1315        return string.Empty;
    11331316      }
    11341317
    11351318      private string GetMaximizationValueOrEmpty(IRun run, string key) {
    1136         return run.Parameters.ContainsKey(key) ? (((BoolValue)run.Parameters[key]).Value ? "MAX" : "MIN") : string.Empty;
     1319        IItem param;
     1320        if (run.Parameters.TryGetValue(key, out param)) {
     1321          var bv = param as BoolValue;
     1322          return bv != null ? (bv.Value ? "MAX" : "MIN") : string.Empty;
     1323        }
     1324        return string.Empty;
    11371325      }
    11381326
    11391327      public override bool Equals(object obj) {
    1140         var other = obj as ProblemDescription;
     1328        var other = obj as ProblemInstance;
    11411329        if (other == null) return false;
    11421330        return ProblemType == other.ProblemType
     
    11551343          (DisplayProblemName ? ProblemName : string.Empty),
    11561344          (DisplayEvaluator ? Evaluator : string.Empty),
    1157           (DisplayMaximization ? Maximization : string.Empty)}.Where(x => !string.IsNullOrEmpty(x)));
     1345          (DisplayMaximization ? Maximization : string.Empty),
     1346          !double.IsNaN(BestKnownQuality) ? BestKnownQuality.ToString(CultureInfo.CurrentCulture.NumberFormat) : string.Empty }.Where(x => !string.IsNullOrEmpty(x)));
     1347      }
     1348
     1349      public event PropertyChangedEventHandler PropertyChanged;
     1350      protected virtual void OnPropertyChanged(string propertyName = null) {
     1351        var handler = PropertyChanged;
     1352        if (handler != null) handler(this, new PropertyChangedEventArgs(propertyName));
    11581353      }
    11591354    }
Note: See TracChangeset for help on using the changeset viewer.