Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
08/16/15 00:53:19 (9 years ago)
Author:
abeham
Message:

#2431:

  • fixed bug in iterated algorithm
  • changed RLD view to use % deviation as targets
  • RLD view now always groups by problem
File:
1 edited

Legend:

Unmodified
Added
Removed
  • branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs

    r12841 r12864  
    220220      }
    221221
    222       var problemTypesDifferent = problems.Select(x => x.ProblemType).Distinct().Count() > 1;
    223       var problemNamesDifferent = problems.Select(x => x.ProblemName).Distinct().Count() > 1;
    224       var evaluatorDifferent = problems.Select(x => x.Evaluator).Distinct().Count() > 1;
     222      var problemTypesDifferent = problems.Select(x => x.ProblemType).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
     223      var problemNamesDifferent = problems.Select(x => x.ProblemName).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
     224      var evaluatorDifferent = problems.Select(x => x.Evaluator).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
    225225      var allEqual = !problemTypesDifferent && !problemNamesDifferent && !evaluatorDifferent;
    226226
    227227      var selectedProblemItem = (ProblemDescription)problemComboBox.SelectedItem;
    228228      problemComboBox.Items.Clear();
     229      problemComboBox.Items.Add(ProblemDescription.MatchAll);
     230      if (selectedProblemItem == null || selectedProblemItem == ProblemDescription.MatchAll)
     231        problemComboBox.SelectedIndex = 0;
    229232      foreach (var prob in problems.OrderBy(x => x.ToString()).ToList()) {
    230233        prob.DisplayProblemType = problemTypesDifferent;
     
    263266    }
    264267
    265     private List<Tuple<string, List<IRun>>> GroupRuns() {
    266       var groupedRuns = new List<Tuple<string, List<IRun>>>();
     268    private Dictionary<string, Dictionary<string, Tuple<double, List<IRun>>>> GroupRuns() {
     269      var groupedRuns = new Dictionary<string, Dictionary<string, Tuple<double, List<IRun>>>>();
    267270
    268271      var table = (string)dataTableComboBox.SelectedItem;
     
    273276
    274277      var selectedProblem = (ProblemDescription)problemComboBox.SelectedItem;
    275       if (selectedProblem == null) selectedProblem = ProblemDescription.MatchAll;
    276 
    277       if (selectedGroup == AllRuns)
    278         groupedRuns = new List<Tuple<string, List<IRun>>> {
    279           Tuple.Create(AllRuns, Content.Where(r => selectedProblem.Match(r) && r.Results.ContainsKey(table) && r.Visible).ToList())
    280         };
    281       else
    282         groupedRuns = (from r in Content
    283                        where r.Parameters.ContainsKey(selectedGroup)
    284                              && selectedProblem.Match(r)
    285                              && r.Results.ContainsKey(table)
    286                              && r.Visible
    287                        group r by r.Parameters[selectedGroup].ToString()
    288                          into g
    289                          select Tuple.Create(g.Key, g.ToList())).ToList();
     278      if (selectedProblem == null) return groupedRuns;
     279
     280      var maximization = IsMaximization();
     281
     282      var targetsPerProblem = (from r in Content
     283                               let pd = new ProblemDescription(r).ToString()
     284                               let target = r.Parameters.ContainsKey("BestKnownQuality")
     285                                && r.Parameters["BestKnownQuality"] is DoubleValue
     286                                  ? ((DoubleValue)r.Parameters["BestKnownQuality"]).Value
     287                                  : ((IndexedDataTable<double>)r.Results[table]).Rows.First().Values.Last().Item2
     288                               group target by pd into g
     289                               select new { Problem = g.Key, Target = maximization ? g.Max() : g.Min() })
     290                              .ToDictionary(x => x.Problem, x => x.Target);
     291
     292      foreach (var x in (from r in Content
     293                         where selectedGroup == AllRuns || r.Parameters.ContainsKey(selectedGroup)
     294                           && selectedProblem.Match(r)
     295                           && r.Results.ContainsKey(table)
     296                           && r.Visible
     297                         group r by selectedGroup == AllRuns ? AllRuns : r.Parameters[selectedGroup].ToString() into g
     298                         select Tuple.Create(g.Key, g.ToList()))) {
     299        var pDict = new Dictionary<string, Tuple<double, List<IRun>>>();
     300        foreach (var y in (from r in x.Item2
     301                           let pd = new ProblemDescription(r).ToString()
     302                           group r by pd into g
     303                           select Tuple.Create(g.Key, g.ToList()))) {
     304          pDict[y.Item1] = Tuple.Create(targetsPerProblem[y.Item1], y.Item2);
     305        }
     306        groupedRuns[x.Item1] = pDict;
     307      }
    290308
    291309      return groupedRuns;
     
    313331      if (string.IsNullOrEmpty(table)) return;
    314332
    315       if (targets == null) GenerateDefaultTargets(table);
     333      if (targets == null) GenerateDefaultTargets();
    316334
    317335      var groupedRuns = GroupRuns();
     
    327345        var maxLength = 0.0;
    328346
    329         foreach (var run in group.Item2) {
    330           var resultsTable = (IndexedDataTable<double>)run.Results[table];
    331           xAxisTitles.Add(resultsTable.VisualProperties.XAxisTitle);
    332 
    333           if (eachOrAllTargetCheckBox.Checked) {
    334             CalculateHitsForEachTarget(hits, resultsTable.Rows.First(), maximization, group.Item2.Count, group.Item1);
    335           } else {
    336             maxLength = CalculateHitsForAllTargets(hits, resultsTable.Rows.First(), maximization, group.Item2.Count, group.Item1);
     347        foreach (var problem in group.Value) {
     348          foreach (var run in problem.Value.Item2) {
     349            var resultsTable = (IndexedDataTable<double>)run.Results[table];
     350            xAxisTitles.Add(resultsTable.VisualProperties.XAxisTitle);
     351
     352            if (eachOrAllTargetCheckBox.Checked) {
     353              CalculateHitsForEachTarget(hits, resultsTable.Rows.First(), maximization, group.Key, group.Value.Count, problem.Value.Item1, problem.Value.Item2.Count);
     354            } else {
     355              maxLength = CalculateHitsForAllTargets(hits, resultsTable.Rows.First(), maximization, group.Key, group.Value.Count, problem.Value.Item1, problem.Value.Item2.Count);
     356            }
    337357          }
    338358        }
     
    369389    }
    370390
    371     private void GenerateDefaultTargets(string table) {
    372       var runs = GroupRuns().SelectMany(x => x.Item2).ToList();
    373       var worst = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item2).Min()).Min();
    374       var best = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item2).Max()).Max();
    375       var maximization = IsMaximization();
    376       if (!maximization) {
    377         var h = worst;
    378         worst = best;
    379         best = h;
    380       }
    381 
    382       if (best == 0 || Math.Abs(best - worst) < Math.Abs(best * 2))
    383         targets = Enumerable.Range(0, 11).Select(x => worst + (x / 10.0) * (best - worst)).ToArray();
    384       else if (best > 0) targets = Enumerable.Range(0, 11).Select(x => best * (1.0 + (10 - x) / 10.0)).ToArray();
    385       else if (best < 0) targets = Enumerable.Range(0, 11).Select(x => best / (1.0 + (10 - x) / 10.0)).ToArray();
    386 
     391    private void GenerateDefaultTargets() {
     392      targets = new[] { 0.1, 0.05, 0.02, 0.01, 0 };
    387393      suppressTargetsEvents = true;
    388       targetsTextBox.Text = string.Join(" ; ", targets);
     394      targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100));
    389395      suppressTargetsEvents = false;
    390396    }
    391397
    392     private void CalculateHitsForEachTarget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, bool maximization, int groupCount, string groupName) {
    393       foreach (var l in targets) {
    394         var key = groupName + "-" + l;
     398    private void CalculateHitsForEachTarget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, bool maximization, string group, int groupCount, double bestTarget, int problemCount) {
     399      foreach (var l in targets.Select(x => (maximization ? (1 - x) : (1 + x)) * bestTarget)) {
     400        var key = group + "-" + l;
    395401        if (!hits.ContainsKey(key)) hits.Add(key, new SortedList<double, double>());
    396402        foreach (var v in row.Values) {
    397403          if (maximization && v.Item2 >= l || !maximization && v.Item2 <= l) {
    398404            if (hits[key].ContainsKey(v.Item1))
    399               hits[key][v.Item1] += 1.0 / groupCount;
    400             else hits[key][v.Item1] = 1.0 / groupCount;
     405              hits[key][v.Item1] += 1.0 / (groupCount * problemCount);
     406            else hits[key][v.Item1] = 1.0 / (groupCount * problemCount);
    401407            break;
    402408          }
     
    405411    }
    406412
    407     private double CalculateHitsForAllTargets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, bool maximization, int groupCount, string groupName) {
     413    private double CalculateHitsForAllTargets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, bool maximization, string group, int groupCount, double bestTarget, int problemCount) {
    408414      var values = row.Values;
    409       if (!hits.ContainsKey(groupName)) hits.Add(groupName, new SortedList<double, double>());
     415      if (!hits.ContainsKey(group)) hits.Add(group, new SortedList<double, double>());
    410416
    411417      var i = 0;
    412418      var j = 0;
    413419      while (i < targets.Length && j < values.Count) {
     420        var target = (maximization ? (1 - targets[i]) : (1 + targets[i])) * bestTarget;
    414421        var current = values[j];
    415         if (maximization && current.Item2 >= targets[i]
    416           || !maximization && current.Item2 <= targets[i]) {
    417           if (!hits[groupName].ContainsKey(current.Item1)) hits[groupName][current.Item1] = 0;
    418           hits[groupName][current.Item1] += 1.0 / (groupCount * targets.Length);
     422        if (maximization && current.Item2 >= target
     423          || !maximization && current.Item2 <= target) {
     424          if (!hits[group].ContainsKey(current.Item1)) hits[group][current.Item1] = 0;
     425          hits[group][current.Item1] += 1.0 / (groupCount * problemCount * targets.Length);
    419426          i++;
    420427        } else {
     
    426433    }
    427434
    428     private void UpdateErtTables(List<Tuple<string, List<IRun>>> groupedRuns) {
     435    private void UpdateErtTables(Dictionary<string, Dictionary<string, Tuple<double, List<IRun>>>> groupedRuns) {
    429436      ertViewHost.Content = null;
    430437      var columns = 1 + targets.Length + 1;
    431       var matrix = new string[groupedRuns.Count + 1, columns];
     438      var matrix = new string[groupedRuns.Count * groupedRuns.Max(x => x.Value.Count) + groupedRuns.Max(x => x.Value.Count), columns];
    432439      var rowCount = 0;
    433440      var maximization = IsMaximization();
     441
     442
     443      var targetsPerProblem = (from r in Content
     444                               let pd = new ProblemDescription(r).ToString()
     445                               let target = r.Parameters.ContainsKey("BestKnownQuality")
     446                                && r.Parameters["BestKnownQuality"] is DoubleValue
     447                                  ? ((DoubleValue)r.Parameters["BestKnownQuality"]).Value
     448                                  : ((IndexedDataTable<double>)r.Results[(string)dataTableComboBox.SelectedItem]).Rows.First().Values.Last().Item2
     449                               group target by pd into g
     450                               select new { Problem = g.Key, Target = maximization ? g.Max() : g.Min() })
     451                              .ToDictionary(x => x.Problem, x => x.Target);
    434452
    435453      var colNames = new string[columns];
    436454      colNames[0] = colNames[colNames.Length - 1] = string.Empty;
    437455      for (var i = 0; i < targets.Length; i++) {
    438         colNames[i + 1] = maximization ? (targets[targets.Length - 1] / targets[i] - 1).ToString("0.0%")
    439                                        : (targets[i] / targets[targets.Length - 1] - 1).ToString("0.0%");
    440         matrix[rowCount, i + 1] = targets[i].ToString(CultureInfo.CurrentCulture.NumberFormat);
    441       }
    442       matrix[rowCount, columns - 1] = "#succ";
    443       rowCount++;
    444 
    445       foreach (var group in groupedRuns) {
    446         matrix[rowCount, 0] = group.Item1;
    447         var bestSucc = string.Empty;
     456        colNames[i + 1] = targets[i].ToString("0.0%");
     457      }
     458      var problems = groupedRuns.SelectMany(x => x.Value.Keys).Distinct().ToList();
     459
     460      foreach (var problem in problems) {
     461        matrix[rowCount, 0] = problem;
    448462        for (var i = 0; i < targets.Length; i++) {
    449           string succ;
    450           matrix[rowCount, i + 1] = CalculateExpectedRunTime(group.Item2, targets[i], maximization, out succ);
    451           if (i == targets.Length - 1) bestSucc = succ;
    452         }
    453         matrix[rowCount, columns - 1] = bestSucc;
     463          matrix[rowCount, i + 1] = (targetsPerProblem[problem] * (maximization ? (1 - targets[i]) : (1 + targets[i]))).ToString(CultureInfo.CurrentCulture.NumberFormat);
     464        }
     465        matrix[rowCount, columns - 1] = "#succ";
    454466        rowCount++;
     467
     468        foreach (var group in groupedRuns) {
     469          matrix[rowCount, 0] = group.Key;
     470          var runs = group.Value[problem].Item2;
     471          var bestSucc = string.Empty;
     472          for (var i = 0; i < targets.Length; i++) {
     473            string succ;
     474            matrix[rowCount, i + 1] = CalculateExpectedRunTime(runs, (maximization ? (1 - targets[i]) : (1 + targets[i])) * group.Value[problem].Item1, maximization, out succ);
     475            if (i == targets.Length - 1) bestSucc = succ;
     476          }
     477          matrix[rowCount, columns - 1] = bestSucc;
     478          rowCount++;
     479        }
    455480      }
    456481      ertViewHost.Content = new StringMatrix(matrix) { ColumnNames = colNames };
     
    500525      var colorCount = 0;
    501526      var lineStyleCount = 0;
    502       var maximization = IsMaximization();
    503527
    504528      foreach (var group in groupedRuns) {
    505529        var hits = new Dictionary<string, SortedList<double, double>>();
    506530
    507         foreach (var run in group.Item2) {
    508           var resultsTable = (IndexedDataTable<double>)run.Results[table];
    509 
    510           if (eachOrAllBudgetsCheckBox.Checked) {
    511             CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), maximization, group.Item2.Count, group.Item1);
    512           } else {
    513             CalculateHitsForAllBudgets(hits, resultsTable.Rows.First(), maximization, group.Item2.Count, group.Item1);
     531        foreach (var problem in group.Value) {
     532          foreach (var run in problem.Value.Item2) {
     533            var resultsTable = (IndexedDataTable<double>)run.Results[table];
     534
     535            if (eachOrAllBudgetsCheckBox.Checked) {
     536              CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), group.Value.Count, group.Key, problem.Value.Item2.Count);
     537            } else {
     538              CalculateHitsForAllBudgets(hits, resultsTable.Rows.First(), group.Value.Count, group.Key, problem.Value.Item2.Count);
     539            }
    514540          }
    515541        }
     
    542568
    543569    private void GenerateDefaultBudgets(string table) {
    544       var runs = GroupRuns().SelectMany(x => x.Item2).ToList();
     570      var runs = GroupRuns().SelectMany(x => x.Value.Values).SelectMany(x => x.Item2).ToList();
    545571      var min = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Min()).Min();
    546572      var max = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Max()).Max();
     
    562588    }
    563589
    564     private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, bool maximization, int groupCount, string groupName) {
     590    private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, string groupName, int problemCount) {
    565591      foreach (var b in budgets) {
    566592        var key = groupName + "-" + b;
     
    573599            var tgt = (prev == null || v.Item1 == b) ? v.Item2 : prev.Item2;
    574600            if (hits[key].ContainsKey(tgt))
    575               hits[key][tgt] += 1.0 / groupCount;
    576             else hits[key][tgt] = 1.0 / groupCount;
     601              hits[key][tgt] += 1.0 / (groupCount * problemCount);
     602            else hits[key][tgt] = 1.0 / (groupCount * problemCount);
    577603            break;
    578604          }
     
    583609    }
    584610
    585     private void CalculateHitsForAllBudgets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, bool maximization, int groupCount, string groupName) {
     611    private void CalculateHitsForAllBudgets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, string groupName, int problemCount) {
    586612      var values = row.Values;
    587613      if (!hits.ContainsKey(groupName)) hits.Add(groupName, new SortedList<double, double>());
     
    596622            var tgt = (prev == null || current.Item1 == budgets[i]) ? current.Item2 : prev.Item2;
    597623            if (!hits[groupName].ContainsKey(tgt)) hits[groupName][tgt] = 0;
    598             hits[groupName][tgt] += 1.0 / (groupCount * budgets.Length);
     624            hits[groupName][tgt] += 1.0 / (groupCount * problemCount * budgets.Length);
    599625          }
    600626          i++;
     
    607633      if (i < budgets.Length && !hits[groupName].ContainsKey(lastTgt)) hits[groupName][lastTgt] = 0;
    608634      while (i < budgets.Length) {
    609         hits[groupName][lastTgt] += 1.0 / (groupCount * budgets.Length);
     635        hits[groupName][lastTgt] += 1.0 / (groupCount * problemCount * budgets.Length);
    610636        i++;
    611637      }
     
    622648    }
    623649    private void problemComboBox_SelectedIndexChanged(object sender, EventArgs e) {
    624       if (dataTableComboBox.SelectedIndex >= 0)
    625         GenerateDefaultTargets((string)dataTableComboBox.SelectedItem);
    626650      UpdateRuns();
    627651      SetEnabledStateOfControls();
     
    643667    private void targetsTextBox_Validating(object sender, CancelEventArgs e) {
    644668      if (suppressTargetsEvents) return;
    645       var targetStrings = targetsTextBox.Text.Split(new[] { ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
    646       var targetList = new List<double>();
     669      var targetStrings = targetsTextBox.Text.Split(new[] { '%', ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
     670      var targetList = new List<decimal>();
    647671      foreach (var ts in targetStrings) {
    648         double t;
    649         if (!double.TryParse(ts, out t)) {
     672        decimal t;
     673        if (!decimal.TryParse(ts, out t)) {
    650674          errorProvider.SetError(targetsTextBox, "Not all targets can be parsed: " + ts);
    651675          e.Cancel = true;
    652676          return;
    653677        }
    654         targetList.Add(t);
     678        targetList.Add(t / 100);
    655679      }
    656680      if (targetList.Count == 0) {
     
    661685      e.Cancel = false;
    662686      errorProvider.SetError(targetsTextBox, null);
    663       targets = targetList.ToArray();
     687      targets = targetList.Select(x => (double)x).ToArray();
    664688      UpdateResultsByTarget();
    665689      SetEnabledStateOfControls();
     
    694718                                  : dialog.Values.Reverse().Select(x => (double)x).ToArray();
    695719            suppressTargetsEvents = true;
    696             targetsTextBox.Text = string.Join(" ; ", targets);
     720            targetsTextBox.Text = string.Join("% ; ", targets);
    697721            suppressTargetsEvents = false;
    698722
     
    707731      var table = (string)dataTableComboBox.SelectedItem;
    708732      var maximization = IsMaximization();
     733
     734      var targetsPerProblem = (from r in Content
     735                               let pd = new ProblemDescription(r).ToString()
     736                               let target = r.Parameters.ContainsKey("BestKnownQuality")
     737                                && r.Parameters["BestKnownQuality"] is DoubleValue
     738                                  ? ((DoubleValue)r.Parameters["BestKnownQuality"]).Value
     739                                  : ((IndexedDataTable<double>)r.Results[table]).Rows.First().Values.Last().Item2
     740                               group target by pd into g
     741                               select new { Problem = g.Key, Target = maximization ? g.Max() : g.Min() })
     742                              .ToDictionary(x => x.Problem, x => x.Target);
     743
    709744      foreach (var run in Content) {
    710745        if (!run.Results.ContainsKey(table)) continue;
     
    713748        var i = 0;
    714749        var j = 0;
     750        var pd = new ProblemDescription(run);
    715751        while (i < targets.Length && j < values.Count) {
     752          var target = (maximization ? (1 - targets[i]) : (1 + targets[i])) * targetsPerProblem[pd.ToString()];
    716753          var current = values[j];
    717           if (maximization && current.Item2 >= targets[i]
    718               || !maximization && current.Item2 <= targets[i]) {
    719             run.Results[table + ".Target" + targets[i]] = new DoubleValue(current.Item1);
     754          if (maximization && current.Item2 >= target
     755              || !maximization && current.Item2 <= target) {
     756            run.Results[table + ".Target" + target] = new DoubleValue(current.Item1);
    720757            i++;
    721758          } else {
     
    865902    private class ProblemDescription {
    866903      private readonly bool matchAll;
    867       public static readonly ProblemDescription MatchAll = new ProblemDescription();
     904      public static readonly ProblemDescription MatchAll = new ProblemDescription() {
     905        ProblemName = "All with Best-Known"
     906      };
    868907
    869908      private ProblemDescription() {
Note: See TracChangeset for help on using the changeset viewer.