Changeset 14647


Ignore:
Timestamp:
02/06/17 22:50:29 (18 months ago)
Author:
abeham
Message:

#2634:

  • fixed RLD analysis for problem instances with best known quality of 0
  • added option to perform comparison on relative targets or absolute targets
  • some performance improvements
Location:
trunk/sources/HeuristicLab.Optimization.Views/3.3/RunCollectionViews
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • trunk/sources/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.Designer.cs

    r14185 r14647  
    6666      this.targetChart = new HeuristicLab.Visualization.ChartControlsExtensions.EnhancedChart();
    6767      this.boundShadingCheckBox = new System.Windows.Forms.CheckBox();
     68      this.targetsRelativeCheckBox = new System.Windows.Forms.CheckBox();
    6869      this.byCostTabPage = new System.Windows.Forms.TabPage();
    6970      this.byCostViewHost = new HeuristicLab.MainForm.WindowsForms.ViewHost();
     
    143144      this.targetsTextBox.Location = new System.Drawing.Point(59, 8);
    144145      this.targetsTextBox.Name = "targetsTextBox";
    145       this.targetsTextBox.Size = new System.Drawing.Size(287, 20);
     146      this.targetsTextBox.Size = new System.Drawing.Size(211, 20);
    146147      this.targetsTextBox.TabIndex = 1;
    147148      this.toolTip.SetToolTip(this.targetsTextBox, "The order of the targets is important, first to-hit targets\r\nshould be given firs" +
     
    221222      this.aggregateTargetsCheckBox.Checked = true;
    222223      this.aggregateTargetsCheckBox.CheckState = System.Windows.Forms.CheckState.Checked;
    223       this.aggregateTargetsCheckBox.Location = new System.Drawing.Point(352, 10);
     224      this.aggregateTargetsCheckBox.Location = new System.Drawing.Point(353, 10);
    224225      this.aggregateTargetsCheckBox.Name = "aggregateTargetsCheckBox";
    225226      this.aggregateTargetsCheckBox.Size = new System.Drawing.Size(74, 17);
     
    249250      this.byTargetTabPage.Controls.Add(this.targetLogScalingCheckBox);
    250251      this.byTargetTabPage.Controls.Add(this.targetsLabel);
     252      this.byTargetTabPage.Controls.Add(this.targetsRelativeCheckBox);
    251253      this.byTargetTabPage.Controls.Add(this.aggregateTargetsCheckBox);
    252254      this.byTargetTabPage.Controls.Add(this.targetsTextBox);
     
    304306      this.boundShadingCheckBox.UseVisualStyleBackColor = true;
    305307      this.boundShadingCheckBox.CheckedChanged += new System.EventHandler(this.boundShadingCheckBox_CheckedChanged);
     308      //
     309      // targetsRelativeCheckBox
     310      //
     311      this.targetsRelativeCheckBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
     312      this.targetsRelativeCheckBox.Appearance = System.Windows.Forms.Appearance.Button;
     313      this.targetsRelativeCheckBox.Checked = true;
     314      this.targetsRelativeCheckBox.CheckState = System.Windows.Forms.CheckState.Checked;
     315      this.targetsRelativeCheckBox.Location = new System.Drawing.Point(276, 6);
     316      this.targetsRelativeCheckBox.Name = "targetsRelativeCheckBox";
     317      this.targetsRelativeCheckBox.Size = new System.Drawing.Size(65, 23);
     318      this.targetsRelativeCheckBox.TabIndex = 2;
     319      this.targetsRelativeCheckBox.Text = "relative";
     320      this.targetsRelativeCheckBox.TextAlign = System.Drawing.ContentAlignment.MiddleCenter;
     321      this.targetsRelativeCheckBox.UseVisualStyleBackColor = true;
     322      this.targetsRelativeCheckBox.CheckedChanged += new System.EventHandler(this.targetsRelativeCheckBox_CheckedChanged);
    306323      //
    307324      // byCostTabPage
     
    473490    private MainForm.WindowsForms.ViewHost byCostViewHost;
    474491    private System.Windows.Forms.CheckBox boundShadingCheckBox;
     492    private System.Windows.Forms.CheckBox targetsRelativeCheckBox;
    475493  }
    476494}
  • trunk/sources/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs

    r14185 r14647  
    3030using HeuristicLab.Analysis;
    3131using HeuristicLab.Collections;
     32using HeuristicLab.Core;
    3233using HeuristicLab.Core.Views;
    3334using HeuristicLab.Data;
     
    7980    private double[] targets;
    8081    private double[] budgets;
     82    private bool targetsAreRelative = true;
     83    private readonly BindingList<ProblemInstance> problems;
    8184
    8285    private bool suppressUpdates;
     
    103106      };
    104107      byCostViewHost.Content = byCostDataTable;
     108      targetsRelativeCheckBox.Checked = targetsAreRelative;
     109      problems = new BindingList<ProblemInstance>();
     110      problemComboBox.DataSource = new BindingSource() { DataSource = problems };
     111      problemComboBox.DataBindings.DefaultDataSourceUpdateMode = DataSourceUpdateMode.OnPropertyChanged;
    105112      suppressUpdates = false;
    106113    }
     
    223230      }
    224231
    225       var problems = new HashSet<ProblemDescription>();
    226       foreach (var run in Content) {
    227         problems.Add(new ProblemDescription(run));
    228       }
    229 
    230       var problemTypesDifferent = problems.Select(x => x.ProblemType).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
    231       var problemNamesDifferent = problems.Select(x => x.ProblemName).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
    232       var evaluatorDifferent = problems.Select(x => x.Evaluator).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
    233       var maximizationDifferent = problems.Select(x => x.Maximization).Distinct().Count() > 1;
     232      var table = (string)dataTableComboBox.SelectedItem;
     233      var problemDict = CalculateBestTargetPerProblemInstance(table);
     234
     235      var problemTypesDifferent = problemDict.Keys.Select(x => x.ProblemType).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
     236      var problemNamesDifferent = problemDict.Keys.Select(x => x.ProblemName).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
     237      var evaluatorDifferent = problemDict.Keys.Select(x => x.Evaluator).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
     238      var maximizationDifferent = problemDict.Keys.Select(x => x.Maximization).Distinct().Count() > 1;
    234239      var allEqual = !problemTypesDifferent && !problemNamesDifferent && !evaluatorDifferent && !maximizationDifferent;
    235240
    236       var selectedProblemItem = (ProblemDescription)problemComboBox.SelectedItem;
    237       problemComboBox.Items.Clear();
    238       problemComboBox.Items.Add(ProblemDescription.MatchAll);
    239       if (selectedProblemItem == null || selectedProblemItem == ProblemDescription.MatchAll)
    240         problemComboBox.SelectedIndex = 0;
    241       foreach (var prob in problems.OrderBy(x => x.ToString()).ToList()) {
    242         prob.DisplayProblemType = problemTypesDifferent;
    243         prob.DisplayProblemName = problemNamesDifferent || allEqual;
    244         prob.DisplayEvaluator = evaluatorDifferent;
    245         prob.DisplayMaximization = maximizationDifferent;
    246         problemComboBox.Items.Add(prob);
    247         if (prob.Equals(selectedProblemItem)) problemComboBox.SelectedItem = prob;
     241      var selectedProblemItem = (ProblemInstance)problemComboBox.SelectedItem;
     242      problems.Clear();
     243      problems.Add(ProblemInstance.MatchAll);
     244      if (problems[0].Equals(selectedProblemItem)) problemComboBox.SelectedItem = problems[0];
     245      foreach (var p in problemDict.ToList()) {
     246        p.Key.BestKnownQuality = p.Value;
     247        p.Key.DisplayProblemType = problemTypesDifferent;
     248        p.Key.DisplayProblemName = problemNamesDifferent || allEqual;
     249        p.Key.DisplayEvaluator = evaluatorDifferent;
     250        p.Key.DisplayMaximization = maximizationDifferent;
     251        problems.Add(p.Key);
     252        if (p.Key.Equals(selectedProblemItem)) problemComboBox.SelectedItem = p.Key;
    248253      }
    249254      SetEnabledStateOfControls();
     
    276281    }
    277282
    278     private Dictionary<string, Dictionary<ProblemDescription, Tuple<double, List<IRun>>>> GroupRuns() {
    279       var groupedRuns = new Dictionary<string, Dictionary<ProblemDescription, Tuple<double, List<IRun>>>>();
     283    private Dictionary<string, Dictionary<ProblemInstance, List<IRun>>> GroupRuns() {
     284      var groupedRuns = new Dictionary<string, Dictionary<ProblemInstance, List<IRun>>>();
    280285
    281286      var table = (string)dataTableComboBox.SelectedItem;
     
    285290      if (string.IsNullOrEmpty(selectedGroup)) return groupedRuns;
    286291
    287       var selectedProblem = (ProblemDescription)problemComboBox.SelectedItem;
     292      var selectedProblem = (ProblemInstance)problemComboBox.SelectedItem;
    288293      if (selectedProblem == null) return groupedRuns;
    289 
    290       var targetsPerProblem = CalculateBestTargetPerProblemInstance(table);
    291 
     294     
    292295      foreach (var x in (from r in Content
    293296                         where (selectedGroup == AllRuns || r.Parameters.ContainsKey(selectedGroup))
     
    297300                         group r by selectedGroup == AllRuns ? AllRuns : r.Parameters[selectedGroup].ToString() into g
    298301                         select Tuple.Create(g.Key, g.ToList()))) {
    299         var pDict = new Dictionary<ProblemDescription, Tuple<double, List<IRun>>>();
    300         foreach (var y in (from r in x.Item2
    301                            let pd = new ProblemDescription(r)
    302                            group r by pd into g
    303                            select Tuple.Create(g.Key, g.ToList()))) {
    304           pDict[y.Item1] = Tuple.Create(targetsPerProblem[y.Item1], y.Item2);
    305         }
    306         groupedRuns[x.Item1] = pDict;
     302        var pDict = problems.ToDictionary(r => r, _ => new List<IRun>());
     303        foreach (var y in x.Item2) {
     304          var pd = new ProblemInstance(y);
     305          List<IRun> l;
     306          if (pDict.TryGetValue(pd, out l)) l.Add(y);
     307        }
     308        groupedRuns[x.Item1] = pDict.Where(a => a.Value.Count > 0).ToDictionary(a => a.Key, a => a.Value);
    307309      }
    308310
     
    333335      // have data for a certain problem instance anymore this is a special case when
    334336      // aggregating over multiple problem instances     
    335       var maxEfforts = new Dictionary<ProblemDescription, double>();
     337      var maxEfforts = new Dictionary<ProblemInstance, double>();
    336338      double minEff = double.MaxValue, maxEff = double.MinValue;
    337339      foreach (var group in groupedRuns) {
     
    341343            problemSpecificMaxEff = 0;
    342344          }
    343           var bestKnownTarget = problem.Value.Item1;
    344345          var max = problem.Key.IsMaximization();
    345           var worstTarget = (max ? (1 - targets.Max()) : (1 + targets.Max())) * bestKnownTarget;
    346           var bestTarget = (max ? (1 - targets.Min()) : (1 + targets.Min())) * bestKnownTarget;
    347           foreach (var run in problem.Value.Item2) {
     346          var absTargets = GetAbsoluteTargetsWorstToBest(max, problem.Key.BestKnownQuality);
     347          var worstTarget = absTargets.First();
     348          var bestTarget = absTargets.Last();
     349          foreach (var run in problem.Value) {
    348350            var row = ((IndexedDataTable<double>)run.Results[table]).Rows.First().Values;
    349351            var a = row.FirstOrDefault(x => max ? x.Item2 >= worstTarget : x.Item2 <= worstTarget);
     
    384386        var noRuns = 0;
    385387        foreach (var problem in group.Value) {
    386           foreach (var run in problem.Value.Item2) {
     388          foreach (var run in problem.Value) {
    387389            var resultsTable = (IndexedDataTable<double>)run.Results[table];
    388390            xAxisTitles.Add(resultsTable.VisualProperties.XAxisTitle);
    389391
    390392            if (aggregateTargetsCheckBox.Checked) {
    391               var length = CalculateHitsForAllTargets(hits, misses, resultsTable.Rows.First(), problem.Key, group.Key, problem.Value.Item1, maxEff);
     393              var length = CalculateHitsForAllTargets(hits, misses, resultsTable.Rows.First(), problem.Key, group.Key, problem.Key.BestKnownQuality, maxEff);
    392394              maxLength = Math.Max(length, maxLength);
    393395            } else {
    394               CalculateHitsForEachTarget(hits, misses, resultsTable.Rows.First(), problem.Key, group.Key, problem.Value.Item1, maxEff);
     396              CalculateHitsForEachTarget(hits, misses, resultsTable.Rows.First(), problem.Key, group.Key, problem.Key.BestKnownQuality, maxEff);
    395397            }
    396398            noRuns++;
     
    407409            IsVisibleInLegend = false,
    408410            ChartType = SeriesChartType.Range,
    409             Color = Color.FromArgb(32, colors[colorCount])
     411            Color = Color.FromArgb(32, colors[colorCount]),
     412            YValuesPerPoint = 2
    410413          };
    411414
     
    475478      }
    476479
    477       if (targets.Length == 1)
    478         targetChart.ChartAreas[0].AxisY.Title = "Probability to be " + (targets[0] * 100) + "% worse than best";
    479       else targetChart.ChartAreas[0].AxisY.Title = "Proportion of reached targets";
     480      if (targets.Length == 1) {
     481        if (targetsAreRelative)
     482          targetChart.ChartAreas[0].AxisY.Title = "Probability to be " + (targets[0] * 100) + "% worse than best";
     483        else targetChart.ChartAreas[0].AxisY.Title = "Probability to reach at least a fitness of " + targets[0];
     484      } else targetChart.ChartAreas[0].AxisY.Title = "Proportion of reached targets";
    480485      targetChart.ChartAreas[0].AxisX.Title = string.Join(" / ", xAxisTitles);
    481486      targetChart.ChartAreas[0].AxisX.IsLogarithmic = CanDisplayLogarithmic();
    482487      targetChart.ChartAreas[0].CursorY.Interval = 0.05;
    483488      UpdateErtTables(groupedRuns);
     489    }
     490   
     491    private IEnumerable<double> GetAbsoluteTargets(bool maximization, double bestKnown) {
     492      if (!targetsAreRelative) return targets;
     493      IEnumerable<double> tmp = null;
     494      if (bestKnown > 0) {
     495        tmp = targets.Select(x => (maximization ? (1 - x) : (1 + x)) * bestKnown);
     496      } else if (bestKnown < 0) {
     497        tmp = targets.Select(x => (!maximization ? (1 - x) : (1 + x)) * bestKnown);
     498      } else {
     499        // relative to 0 is impossible
     500        tmp = targets;
     501      }
     502      return tmp;
     503    }
     504
     505    private double[] GetAbsoluteTargetsWorstToBest(bool maximization, double bestKnown) {
     506      var absTargets = GetAbsoluteTargets(maximization, bestKnown);
     507      return maximization ? absTargets.OrderBy(x => x).ToArray() : absTargets.OrderByDescending(x => x).ToArray();
    484508    }
    485509
     
    493517    private void CalculateHitsForEachTarget(Dictionary<string, SortedList<double, int>> hits,
    494518                                            Dictionary<string, SortedList<double, int>> misses,
    495                                             IndexedDataRow<double> row, ProblemDescription problem,
     519                                            IndexedDataRow<double> row, ProblemInstance problem,
    496520                                            string group, double bestTarget, double maxEffort) {
    497       foreach (var t in targets.Select(x => Tuple.Create((problem.IsMaximization() ? (1 - x) : (1 + x)) * bestTarget, x))) {
     521      var maximization = problem.IsMaximization();
     522      foreach (var t in targets.Zip(GetAbsoluteTargets(maximization, bestTarget), (rt, at) => Tuple.Create(at, rt))) {
    498523        var l = t.Item1;
    499         var key = group + "_" + (t.Item2 * 100) + "%_" + l;
     524        var key = group + "_" + (targetsAreRelative ? (t.Item2 * 100) + "%_" + l : l.ToString());
    500525        if (!hits.ContainsKey(key)) {
    501526          hits.Add(key, new SortedList<double, int>());
     
    505530        foreach (var v in row.Values) {
    506531          if (v.Item1 > maxEffort) break;
    507           if (problem.IsMaximization() && v.Item2 >= l || !problem.IsMaximization() && v.Item2 <= l) {
     532          if (maximization && v.Item2 >= l || !maximization && v.Item2 <= l) {
    508533            if (hits[key].ContainsKey(v.Item1))
    509534              hits[key][v.Item1]++;
     
    524549    private double CalculateHitsForAllTargets(Dictionary<string, SortedList<double, int>> hits,
    525550                                              Dictionary<string, SortedList<double, int>> misses,
    526                                               IndexedDataRow<double> row, ProblemDescription problem,
     551                                              IndexedDataRow<double> row, ProblemInstance problem,
    527552                                              string group, double bestTarget, double maxEffort) {
    528553      var values = row.Values;
     
    534559      var i = 0;
    535560      var j = 0;
    536       while (i < targets.Length && j < values.Count) {
    537         var target = (problem.IsMaximization() ? (1 - targets[i]) : (1 + targets[i])) * bestTarget;
     561      var max = problem.IsMaximization();
     562      var absTargets = GetAbsoluteTargetsWorstToBest(max, bestTarget);
     563      while (i < absTargets.Length && j < values.Count) {
     564        var target = absTargets[i];
    538565        var current = values[j];
    539566        if (current.Item1 > maxEffort) break;
     
    549576      if (j == values.Count) j--;
    550577      var effort = Math.Min(values[j].Item1, maxEffort);
    551       if (i < targets.Length) {
     578      if (i < absTargets.Length) {
    552579        if (misses[group].ContainsKey(effort))
    553           misses[group][effort] += targets.Length - i;
    554         else misses[group][effort] = targets.Length - i;
     580          misses[group][effort] += absTargets.Length - i;
     581        else misses[group][effort] = absTargets.Length - i;
    555582      }
    556583      return effort;
    557584    }
    558585
    559     private void UpdateErtTables(Dictionary<string, Dictionary<ProblemDescription, Tuple<double, List<IRun>>>> groupedRuns) {
     586    private void UpdateErtTables(Dictionary<string, Dictionary<ProblemInstance, List<IRun>>> groupedRuns) {
    560587      ertTableView.Content = null;
    561588      var columns = 1 + targets.Length + 1;
     
    565592      var tableName = (string)dataTableComboBox.SelectedItem;
    566593      if (string.IsNullOrEmpty(tableName)) return;
    567 
    568       var targetsPerProblem = CalculateBestTargetPerProblemInstance(tableName);
    569 
    570       var colNames = new string[columns];
    571       colNames[0] = colNames[colNames.Length - 1] = string.Empty;
    572       for (var i = 0; i < targets.Length; i++) {
    573         colNames[i + 1] = targets[i].ToString("0.0%");
    574       }
     594     
    575595      var problems = groupedRuns.SelectMany(x => x.Value.Keys).Distinct().ToList();
    576596
    577597      foreach (var problem in problems) {
     598        var max = problem.IsMaximization();
    578599        matrix[rowCount, 0] = problem.ToString();
    579         for (var i = 0; i < targets.Length; i++) {
    580           matrix[rowCount, i + 1] = (targetsPerProblem[problem] * (problem.IsMaximization() ? (1 - targets[i]) : (1 + targets[i]))).ToString(CultureInfo.CurrentCulture.NumberFormat);
     600        var absTargets = GetAbsoluteTargetsWorstToBest(max, problem.BestKnownQuality);
     601        for (var i = 0; i < absTargets.Length; i++) {
     602          matrix[rowCount, i + 1] = absTargets[i].ToString(CultureInfo.CurrentCulture.NumberFormat);
    581603        }
    582604        matrix[rowCount, columns - 1] = "#succ";
     
    590612            continue;
    591613          }
    592           var runs = group.Value[problem].Item2;
    593           ErtCalculationResult result = default(ErtCalculationResult);
    594           for (var i = 0; i < targets.Length; i++) {
    595             result = ExpectedRuntimeHelper.CalculateErt(runs, tableName, (problem.IsMaximization() ? (1 - targets[i]) : (1 + targets[i])) * group.Value[problem].Item1, problem.IsMaximization());
     614          var runs = group.Value[problem];
     615          var result = default(ErtCalculationResult);
     616          for (var i = 0; i < absTargets.Length; i++) {
     617            result = ExpectedRuntimeHelper.CalculateErt(runs, tableName, absTargets[i], max);
    596618            matrix[rowCount, i + 1] = result.ToString();
    597619          }
     
    600622        }
    601623      }
    602       ertTableView.Content = new StringMatrix(matrix) { ColumnNames = colNames };
     624      ertTableView.Content = new StringMatrix(matrix);
    603625      ertTableView.DataGridView.AutoResizeColumns(DataGridViewAutoSizeColumnsMode.AllCells);
    604626    }
     
    621643      var colorCount = 0;
    622644      var lineStyleCount = 0;
    623 
    624       var targetsPerProblem = CalculateBestTargetPerProblemInstance((string)dataTableComboBox.SelectedItem);
    625 
     645     
    626646      foreach (var group in groupedRuns) {
    627647        var hits = new Dictionary<string, SortedList<double, double>>();
    628648
    629649        foreach (var problem in group.Value) {
    630           foreach (var run in problem.Value.Item2) {
     650          foreach (var run in problem.Value) {
    631651            var resultsTable = (IndexedDataTable<double>)run.Results[table];
    632652
    633653            if (eachOrAllBudgetsCheckBox.Checked) {
    634               CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), group.Value.Count, problem.Key, group.Key, problem.Value.Item2.Count, targetsPerProblem[problem.Key]);
     654              CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), group.Value.Count, problem.Key, group.Key, problem.Value.Count);
    635655            } else {
    636               CalculateHitsForAllBudgets(hits, resultsTable.Rows.First(), group.Value.Count, problem.Key, group.Key, problem.Value.Item2.Count, targetsPerProblem[problem.Key]);
     656              CalculateHitsForAllBudgets(hits, resultsTable.Rows.First(), group.Value.Count, problem.Key, group.Key, problem.Value.Count);
    637657            }
    638658          }
     
    667687
    668688    private void GenerateDefaultBudgets(string table) {
    669       var runs = GroupRuns().SelectMany(x => x.Value.Values).SelectMany(x => x.Item2).ToList();
     689      var runs = GroupRuns().SelectMany(x => x.Value.Values).SelectMany(x => x).ToList();
    670690      var min = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Min()).Min();
    671691      var max = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Max()).Max();
     
    687707    }
    688708
    689     private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, ProblemDescription problem, string groupName, int problemCount, double bestTarget) {
     709    private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, ProblemInstance problem, string groupName, int problemCount) {
     710      var max = problem.IsMaximization();
    690711      foreach (var b in budgets) {
    691712        var key = groupName + "-" + b;
     
    697718            if (prev == null && v.Item1 != b) break;
    698719            var tgt = ((prev == null || v.Item1 == b) ? v.Item2 : prev.Item2);
    699             tgt = problem.IsMaximization() ? bestTarget / tgt : tgt / bestTarget;
    700             if (hits[key].ContainsKey(tgt))
    701               hits[key][tgt] += 1.0 / (groupCount * problemCount);
    702             else hits[key][tgt] = 1.0 / (groupCount * problemCount);
     720            var relTgt = CalculateRelativeDifference(max, problem.BestKnownQuality, tgt);
     721            if (hits[key].ContainsKey(relTgt))
     722              hits[key][relTgt] += 1.0 / (groupCount * problemCount);
     723            else hits[key][relTgt] = 1.0 / (groupCount * problemCount);
    703724            break;
    704725          }
     
    709730    }
    710731
    711     private void CalculateHitsForAllBudgets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, ProblemDescription problem, string groupName, int problemCount, double bestTarget) {
     732    private void CalculateHitsForAllBudgets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, ProblemInstance problem, string groupName, int problemCount) {
    712733      var values = row.Values;
    713734      if (!hits.ContainsKey(groupName)) hits.Add(groupName, new SortedList<double, double>());
     
    716737      var j = 0;
    717738      Tuple<double, double> prev = null;
     739      var max = problem.IsMaximization();
    718740      while (i < budgets.Length && j < values.Count) {
    719741        var current = values[j];
     
    721743          if (prev != null || current.Item1 == budgets[i]) {
    722744            var tgt = (prev == null || current.Item1 == budgets[i]) ? current.Item2 : prev.Item2;
    723             tgt = problem.IsMaximization() ? bestTarget / tgt : tgt / bestTarget;
    724             if (!hits[groupName].ContainsKey(tgt)) hits[groupName][tgt] = 0;
    725             hits[groupName][tgt] += 1.0 / (groupCount * problemCount * budgets.Length);
     745            var relTgt = CalculateRelativeDifference(max, problem.BestKnownQuality, tgt);
     746            if (!hits[groupName].ContainsKey(relTgt)) hits[groupName][relTgt] = 0;
     747            hits[groupName][relTgt] += 1.0 / (groupCount * problemCount * budgets.Length);
    726748          }
    727749          i++;
     
    732754      }
    733755      var lastTgt = values.Last().Item2;
    734       lastTgt = problem.IsMaximization() ? bestTarget / lastTgt : lastTgt / bestTarget;
    735       if (i < budgets.Length && !hits[groupName].ContainsKey(lastTgt)) hits[groupName][lastTgt] = 0;
     756      var lastRelTgt = CalculateRelativeDifference(max, problem.BestKnownQuality, lastTgt);
     757      if (i < budgets.Length && !hits[groupName].ContainsKey(lastRelTgt)) hits[groupName][lastRelTgt] = 0;
    736758      while (i < budgets.Length) {
    737         hits[groupName][lastTgt] += 1.0 / (groupCount * problemCount * budgets.Length);
     759        hits[groupName][lastRelTgt] += 1.0 / (groupCount * problemCount * budgets.Length);
    738760        i++;
    739761      }
     
    756778      if (dataTableComboBox.SelectedIndex >= 0)
    757779        GenerateDefaultBudgets((string)dataTableComboBox.SelectedItem);
     780      UpdateBestKnownQualities();
    758781      UpdateRuns();
    759782      SetEnabledStateOfControls();
     
    782805          return;
    783806        }
    784         targetList.Add(t / 100);
     807        if (targetsAreRelative)
     808          targetList.Add(t / 100);
     809        else targetList.Add(t);
    785810      }
    786811      if (targetList.Count == 0) {
     
    791816      e.Cancel = false;
    792817      errorProvider.SetError(targetsTextBox, null);
    793       targets = targetList.Select(x => (double)x).OrderByDescending(x => x).ToArray();
     818      targets = targetsAreRelative ? targetList.Select(x => (double)x).OrderByDescending(x => x).ToArray() : targetList.Select(x => (double)x).ToArray();
    794819      suppressTargetsEvents = true;
    795       try { targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100)) + "%"; } finally { suppressTargetsEvents = false; }
     820      try {
     821        if (targetsAreRelative)
     822          targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100)) + "%";
     823        else targetsTextBox.Text = string.Join(" ; ", targets);
     824      } finally { suppressTargetsEvents = false; }
    796825
    797826      UpdateResultsByTarget();
     
    804833        UpdateResultsByTarget();
    805834      } finally { ResumeRepaint(true); }
     835    }
     836
     837    private void targetsRelativeCheckBox_CheckedChanged(object sender, EventArgs e) {
     838      targetsRelativeCheckBox.Text = targetsRelativeCheckBox.Checked ? "relative" : "absolute";
     839      if (suppressUpdates) return;
     840      var pd = (ProblemInstance)problemComboBox.SelectedItem;
     841      if (!double.IsNaN(pd.BestKnownQuality)) {
     842        var max = pd.IsMaximization();
     843        if (targetsAreRelative) targets = GetAbsoluteTargets(max, pd.BestKnownQuality).ToArray();
     844        else {
     845          // Rounding to 5 digits since it's certainly appropriate for this application
     846          if (pd.BestKnownQuality > 0) {
     847            targets = targets.Select(x => Math.Round(max ? 1.0 - (x / pd.BestKnownQuality) : (x / pd.BestKnownQuality) - 1.0, 5)).ToArray();
     848          } else if (pd.BestKnownQuality < 0) {
     849            targets = targets.Select(x => Math.Round(!max ? 1.0 - (x / pd.BestKnownQuality) : (x / pd.BestKnownQuality) - 1.0, 5)).ToArray();
     850          }
     851        }
     852      }
     853      targetsAreRelative = targetsRelativeCheckBox.Checked;
     854      SuspendRepaint();
     855      suppressTargetsEvents = true;
     856      try {
     857        if (targetsAreRelative)
     858          targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100)) + "%";
     859        else targetsTextBox.Text = string.Join(" ; ", targets);
     860
     861        UpdateResultsByTarget();
     862      } finally { suppressTargetsEvents = false; ResumeRepaint(true); }
    806863    }
    807864
     
    823880            targets = dialog.Values.Select(x => (double)x).ToArray();
    824881            suppressTargetsEvents = true;
    825             targetsTextBox.Text = string.Join("% ; ", targets);
     882            if (targetsAreRelative)
     883              targetsTextBox.Text = string.Join("% ; ", targets);
     884            else targetsTextBox.Text = string.Join(" ; ", targets);
    826885            suppressTargetsEvents = false;
    827886
     
    835894    private void addTargetsAsResultButton_Click(object sender, EventArgs e) {
    836895      var table = (string)dataTableComboBox.SelectedItem;
    837 
    838       var targetsPerProblem = CalculateBestTargetPerProblemInstance(table);
     896      if (string.IsNullOrEmpty(table)) return;
     897
     898      var targetsPerProblem = problems.ToDictionary(x => x, x => x.BestKnownQuality);
    839899
    840900      foreach (var run in Content) {
     
    844904        var i = 0;
    845905        var j = 0;
    846         var pd = new ProblemDescription(run);
    847         while (i < targets.Length && j < values.Count) {
    848           var target = (pd.IsMaximization() ? (1 - targets[i]) : (1 + targets[i])) * targetsPerProblem[pd];
     906        var pd = new ProblemInstance(run);
     907        var max = pd.IsMaximization();
     908        var absTargets = GetAbsoluteTargetsWorstToBest(max, targetsPerProblem[pd]);
     909        while (i < absTargets.Length && j < values.Count) {
     910          var target = absTargets[i];
    849911          var current = values[j];
    850           if (pd.IsMaximization() && current.Item2 >= target
    851               || !pd.IsMaximization() && current.Item2 <= target) {
     912          if (max && current.Item2 >= target || !max && current.Item2 <= target) {
    852913            run.Results[table + ".Target" + target] = new DoubleValue(current.Item1);
    853914            i++;
     
    9671028
    9681029    #region Helpers
    969     private Dictionary<ProblemDescription, double> CalculateBestTargetPerProblemInstance(string table) {
     1030    private double CalculateRelativeDifference(bool maximization, double bestKnown, double fit) {
     1031      if (bestKnown == 0) {
     1032        // no relative difference with respect to bestKnown possible
     1033        return maximization ? -fit : fit;
     1034      }
     1035      var absDiff = (fit - bestKnown);
     1036      var relDiff = absDiff / bestKnown;
     1037      if (maximization) {
     1038        return bestKnown > 0 ? -relDiff : relDiff;
     1039      } else {
     1040        return bestKnown > 0 ? relDiff : -relDiff;
     1041      }
     1042    }
     1043
     1044    private Dictionary<ProblemInstance, double> CalculateBestTargetPerProblemInstance(string table) {
     1045      if (table == null) table = string.Empty;
    9701046      return (from r in Content
    9711047              where r.Visible
    972               let pd = new ProblemDescription(r)
     1048              let pd = new ProblemInstance(r)
    9731049              let target = r.Parameters.ContainsKey("BestKnownQuality")
    9741050                           && r.Parameters["BestKnownQuality"] is DoubleValue
    9751051                ? ((DoubleValue)r.Parameters["BestKnownQuality"]).Value
    976                 : ((IndexedDataTable<double>)r.Results[table]).Rows.First().Values.Last().Item2
     1052                : (r.Results.ContainsKey(table) && r.Results[table] is IndexedDataTable<double>
     1053                  ? ((IndexedDataTable<double>)r.Results[table]).Rows.First().Values.Last().Item2
     1054                  : double.NaN)
    9771055              group target by pd into g
    978               select new { Problem = g.Key, Target = g.Key.IsMaximization() ? g.Max() : g.Min() })
     1056              select new { Problem = g.Key, Target = g.Any(x => !double.IsNaN(x)) ? (g.Key.IsMaximization() ? g.Max() : g.Where(x => !double.IsNaN(x)).Min()) : double.NaN })
    9791057        .ToDictionary(x => x.Problem, x => x.Target);
    9801058    }
     
    9901068        UpdateResultsByCost();
    9911069      } finally { ResumeRepaint(true); }
     1070    }
     1071
     1072    private void UpdateBestKnownQualities() {
     1073      var table = (string)dataTableComboBox.SelectedItem;
     1074      if (string.IsNullOrEmpty(table)) return;
     1075
     1076      var targetsPerProblem = CalculateBestTargetPerProblemInstance(table);
     1077      foreach (var pd in problems) {
     1078        double bkq;
     1079        if (targetsPerProblem.TryGetValue(pd, out bkq))
     1080          pd.BestKnownQuality = bkq;
     1081        else pd.BestKnownQuality = double.NaN;
     1082      }
     1083      //problemComboBox.ResetBindings();
    9921084    }
    9931085    #endregion
     
    10781170    }
    10791171
    1080     private class ProblemDescription {
     1172    private class ProblemInstance : INotifyPropertyChanged {
    10811173      private readonly bool matchAll;
    1082       public static readonly ProblemDescription MatchAll = new ProblemDescription() {
     1174      public static readonly ProblemInstance MatchAll = new ProblemInstance() {
    10831175        ProblemName = "All with Best-Known"
    10841176      };
    10851177
    1086       private ProblemDescription() {
     1178      private ProblemInstance() {
    10871179        ProblemType = string.Empty;
    10881180        ProblemName = string.Empty;
     
    10941186        DisplayMaximization = false;
    10951187        matchAll = true;
    1096       }
    1097 
    1098       public ProblemDescription(IRun run) {
     1188        BestKnownQuality = double.NaN;
     1189      }
     1190
     1191      public ProblemInstance(IRun run) {
    10991192        ProblemType = GetStringValueOrEmpty(run, "Problem Type");
    11001193        ProblemName = GetStringValueOrEmpty(run, "Problem Name");
     
    11061199        DisplayMaximization = !string.IsNullOrEmpty(Maximization);
    11071200        matchAll = false;
    1108       }
    1109 
    1110       public bool DisplayProblemType { get; set; }
    1111       public string ProblemType { get; set; }
    1112       public bool DisplayProblemName { get; set; }
    1113       public string ProblemName { get; set; }
    1114       public bool DisplayEvaluator { get; set; }
    1115       public string Evaluator { get; set; }
    1116       public bool DisplayMaximization { get; set; }
    1117       public string Maximization { get; set; }
     1201        BestKnownQuality = GetDoubleValueOrNaN(run, "BestKnownQuality");
     1202      }
     1203
     1204      private bool displayProblemType;
     1205      public bool DisplayProblemType {
     1206        get { return displayProblemType; }
     1207        set {
     1208          if (displayProblemType == value) return;
     1209          displayProblemType = value;
     1210          OnPropertyChanged("DisplayProblemType");
     1211        }
     1212      }
     1213      private string problemType;
     1214      public string ProblemType {
     1215        get { return problemType; }
     1216        set {
     1217          if (problemType == value) return;
     1218          problemType = value;
     1219          OnPropertyChanged("ProblemType");
     1220        }
     1221      }
     1222      private bool displayProblemName;
     1223      public bool DisplayProblemName {
     1224        get { return displayProblemName; }
     1225        set {
     1226          if (displayProblemName == value) return;
     1227          displayProblemName = value;
     1228          OnPropertyChanged("DisplayProblemName");
     1229        }
     1230      }
     1231      private string problemName;
     1232      public string ProblemName {
     1233        get { return problemName; }
     1234        set {
     1235          if (problemName == value) return;
     1236          problemName = value;
     1237          OnPropertyChanged("ProblemName");
     1238        }
     1239      }
     1240      private bool displayEvaluator;
     1241      public bool DisplayEvaluator {
     1242        get { return displayEvaluator; }
     1243        set {
     1244          if (displayEvaluator == value) return;
     1245          displayEvaluator = value;
     1246          OnPropertyChanged("DisplayEvaluator");
     1247        }
     1248      }
     1249      private string evaluator;
     1250      public string Evaluator {
     1251        get { return evaluator; }
     1252        set {
     1253          if (evaluator == value) return;
     1254          evaluator = value;
     1255          OnPropertyChanged("Evaluator");
     1256        }
     1257      }
     1258      private bool displayMaximization;
     1259      public bool DisplayMaximization {
     1260        get { return displayMaximization; }
     1261        set {
     1262          if (displayMaximization == value) return;
     1263          displayMaximization = value;
     1264          OnPropertyChanged("DisplayMaximization");
     1265        }
     1266      }
     1267      private string maximization;
     1268      public string Maximization {
     1269        get { return maximization; }
     1270        set {
     1271          if (maximization == value) return;
     1272          maximization = value;
     1273          OnPropertyChanged("Maximization");
     1274        }
     1275      }
     1276      private double bestKnownQuality;
     1277      public double BestKnownQuality {
     1278        get { return bestKnownQuality; }
     1279        set {
     1280          if (bestKnownQuality == value) return;
     1281          bestKnownQuality = value;
     1282          OnPropertyChanged("BestKnownQuality");
     1283        }
     1284      }
    11181285
    11191286      public bool IsMaximization() {
     
    11291296      }
    11301297
     1298      private double GetDoubleValueOrNaN(IRun run, string key) {
     1299        IItem param;
     1300        if (run.Parameters.TryGetValue(key, out param)) {
     1301          var dv = param as DoubleValue;
     1302          return dv != null ? dv.Value : double.NaN;
     1303        }
     1304        return double.NaN;
     1305      }
     1306
    11311307      private string GetStringValueOrEmpty(IRun run, string key) {
    1132         return run.Parameters.ContainsKey(key) ? ((StringValue)run.Parameters[key]).Value : string.Empty;
     1308        IItem param;
     1309        if (run.Parameters.TryGetValue(key, out param)) {
     1310          var sv = param as StringValue;
     1311          return sv != null ? sv.Value : string.Empty;
     1312        }
     1313        return string.Empty;
    11331314      }
    11341315
    11351316      private string GetMaximizationValueOrEmpty(IRun run, string key) {
    1136         return run.Parameters.ContainsKey(key) ? (((BoolValue)run.Parameters[key]).Value ? "MAX" : "MIN") : string.Empty;
     1317        IItem param;
     1318        if (run.Parameters.TryGetValue(key, out param)) {
     1319          var bv = param as BoolValue;
     1320          return bv != null ? (bv.Value ? "MAX" : "MIN") : string.Empty;
     1321        }
     1322        return string.Empty;
    11371323      }
    11381324
    11391325      public override bool Equals(object obj) {
    1140         var other = obj as ProblemDescription;
     1326        var other = obj as ProblemInstance;
    11411327        if (other == null) return false;
    11421328        return ProblemType == other.ProblemType
     
    11551341          (DisplayProblemName ? ProblemName : string.Empty),
    11561342          (DisplayEvaluator ? Evaluator : string.Empty),
    1157           (DisplayMaximization ? Maximization : string.Empty)}.Where(x => !string.IsNullOrEmpty(x)));
     1343          (DisplayMaximization ? Maximization : string.Empty),
     1344          !double.IsNaN(BestKnownQuality) ? BestKnownQuality.ToString(CultureInfo.CurrentCulture.NumberFormat) : string.Empty }.Where(x => !string.IsNullOrEmpty(x)));
     1345      }
     1346
     1347      public event PropertyChangedEventHandler PropertyChanged;
     1348      protected virtual void OnPropertyChanged(string propertyName = null) {
     1349        var handler = PropertyChanged;
     1350        if (handler != null) handler(this, new PropertyChangedEventArgs(propertyName));
    11581351      }
    11591352    }
Note: See TracChangeset for help on using the changeset viewer.