Changeset 15676


Ignore:
Timestamp:
01/30/18 15:55:00 (19 months ago)
Author:
abeham
Message:

#1614:

  • Refactored RLD view, added additional classes
File:
1 edited

Legend:

Unmodified
Added
Removed
  • branches/GeneralizedQAP/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs

    r15615 r15676  
    2121
    2222using System;
     23using System.Collections;
    2324using System.Collections.Generic;
    2425using System.ComponentModel;
     
    7879    }
    7980
     81    private List<AlgorithmInstance> groups;
     82
    8083    private double[] targets;
    8184    private double[] budgets;
     
    141144      UpdateGroupAndProblemComboBox();
    142145      UpdateDataTableComboBox();
     146      GroupRuns();
    143147      foreach (var run in e.Items)
    144148        RegisterRunEvents(run);
     
    152156      UpdateGroupAndProblemComboBox();
    153157      UpdateDataTableComboBox();
     158      GroupRuns();
    154159      foreach (var run in e.Items)
    155160        DeregisterRunEvents(run);
     
    163168      UpdateGroupAndProblemComboBox();
    164169      UpdateDataTableComboBox();
     170      GroupRuns();
    165171      foreach (var run in e.OldItems)
    166172        DeregisterRunEvents(run);
     
    215221        UpdateGroupAndProblemComboBox();
    216222        UpdateDataTableComboBox();
     223        GroupRuns();
    217224      }
    218225    }
     
    258265        if (p.Key.Equals(selectedProblemItem)) problemComboBox.SelectedItem = p.Key;
    259266      }
     267
    260268      SetEnabledStateOfControls();
    261269    }
     
    288296    }
    289297
    290     private IEnumerable<AlgorithmInstance> GroupRuns() {
     298    private void GroupRuns() {
     299      groups = new List<AlgorithmInstance>();
     300
    291301      var table = (string)dataTableComboBox.SelectedItem;
    292       if (string.IsNullOrEmpty(table)) yield break;
     302      if (string.IsNullOrEmpty(table)) return;
    293303
    294304      var selectedGroup = (string)groupComboBox.SelectedItem;
    295       if (string.IsNullOrEmpty(selectedGroup)) yield break;
     305      if (string.IsNullOrEmpty(selectedGroup)) return;
    296306
    297307      var selectedProblem = (ProblemInstance)problemComboBox.SelectedItem;
    298       if (selectedProblem == null) yield break;
    299      
    300       foreach (var x in (from r in Content
    301                          where (selectedGroup == AllInstances || r.Parameters.ContainsKey(selectedGroup))
    302                            && selectedProblem.Match(r)
    303                            && r.Results.ContainsKey(table)
    304                            && r.Visible
    305                          let key = selectedGroup == AllInstances ? AllInstances : r.Parameters[selectedGroup].ToString()
    306                          group r by key into g
    307                          select new AlgorithmInstance(g.Key, g, problems))) {
    308         yield return x;
     308      if (selectedProblem == null) return;
     309
     310      foreach (var alg in from r in Content
     311                          where (selectedGroup == AllInstances || r.Parameters.ContainsKey(selectedGroup))
     312                            && r.Visible
     313                            && selectedProblem.Match(r)
     314                            && r.Results.ContainsKey(table)
     315                            && r.Results[table] is IndexedDataTable<double>
     316                          let key = selectedGroup == AllInstances ? AllInstances : r.Parameters[selectedGroup].ToString()
     317                          group r by key into g
     318                          select g) {
     319        var trials = from run in alg
     320                     from pd in problems
     321                     where pd.Equals(new ProblemInstance(run))
     322                     let cgraph = ((IndexedDataTable<double>)run.Results[table])
     323                     group cgraph by pd into g
     324                     select g;
     325
     326        groups.Add(new AlgorithmInstance(alg.Key, trials));
    309327      }
    310328    }
     
    321339
    322340      if (targets == null) GenerateDefaultTargets();
    323 
    324       var algInstances = GroupRuns().ToList();
    325       if (algInstances.Count == 0) return;
     341     
     342      if (groups.Count == 0) return;
    326343
    327344      var xAxisTitles = new HashSet<string>();
     
    337354      var aggregate = aggregateTargetsCheckBox.Checked;
    338355      double minEff = double.MaxValue, maxEff = double.MinValue;
    339       foreach (var alg in algInstances) {
     356      foreach (var alg in groups) {
    340357        var noRuns = 0;
    341358        SortedList<double, int> epdfHits = null, epdfMisses = null;
     
    344361          misses[alg.Name] = epdfMisses = new SortedList<double, int>();
    345362        }
    346         foreach (var problem in alg.GetProblemInstances()) {
     363        foreach (var problem in alg.GetCases()) {
    347364          var max = problem.IsMaximization();
    348365          var absTargets = GetAbsoluteTargets(problem).ToArray();
    349           foreach (var run in alg.GetRuns(problem)) {
     366          foreach (var run in alg.GetTrials(problem)) {
    350367            noRuns++;
    351             var resultsTable = (IndexedDataTable<double>)run.Results[table];
    352             xAxisTitles.Add(resultsTable.VisualProperties.XAxisTitle);
    353 
    354             var efforts = absTargets.Select(t => GetEffortToHitTarget(resultsTable.Rows.First().Values, t, max)).ToArray();
     368            xAxisTitles.Add(run.XAxisName);
     369
     370            var efforts = absTargets.Select(t => GetEffortToHitTarget(run, t, max)).ToArray();
    355371            minEff = Math.Min(minEff, efforts.Min(x => x.Item2));
    356372            maxEff = Math.Max(maxEff, efforts.Max(x => x.Item2));
     
    391407      targetChart.ChartAreas[0].CursorY.Interval = 0.05;
    392408
    393       UpdateErtTables(algInstances);
     409      UpdateErtTables();
    394410    }
    395411
     
    582598
    583599    private Tuple<bool, double> GetEffortToHitTarget(
    584         ObservableList<Tuple<double, double>> convergenceGraph,
     600        ConvergenceGraph cgraph,
    585601        double absTarget, bool maximization) {
    586       if (convergenceGraph.Count == 0)
     602      if (cgraph.Points == 0)
    587603        throw new ArgumentException("Convergence graph is empty.", "convergenceGraph");
    588604     
    589       var index = convergenceGraph.BinarySearch(Tuple.Create(0.0, absTarget), new TargetComparer(maximization));
     605      var index = cgraph.BinarySearch(new ConvergenceGraphPoint(0.0, absTarget), new TargetComparer(maximization));
    590606      if (index >= 0) {
    591         return Tuple.Create(true, convergenceGraph[index].Item1);
     607        return Tuple.Create(true, cgraph[index].Runlength);
    592608      } else {
    593609        index = ~index;
    594         if (index >= convergenceGraph.Count)
    595           return Tuple.Create(false, convergenceGraph.Last().Item1);
    596         return Tuple.Create(true, convergenceGraph[index].Item1);
    597       }
    598     }
    599 
    600     private void UpdateErtTables(List<AlgorithmInstance> algorithmInstances) {
     610        if (index >= cgraph.Points)
     611          return Tuple.Create(false, cgraph.Last().Runlength);
     612        return Tuple.Create(true, cgraph[index].Runlength);
     613      }
     614    }
     615
     616    private void UpdateErtTables() {
    601617      ertTableView.Content = null;
    602618      var columns = targets.Length + 1;
    603       var totalRows = algorithmInstances.Count * algorithmInstances.Max(x => x.GetNumberOfProblemInstances()) + algorithmInstances.Max(x => x.GetNumberOfProblemInstances());
     619      var totalRows = groups.Count * groups.Max(x => x.GetNumberOfCases()) + groups.Max(x => x.GetNumberOfCases());
    604620      var matrix = new StringMatrix(totalRows, columns);
    605621      var rowNames = new List<string>();
     
    611627      if (string.IsNullOrEmpty(tableName)) return;
    612628     
    613       var problems = algorithmInstances.SelectMany(x => x.GetProblemInstances()).Distinct().ToList();
     629      var problems = groups.SelectMany(x => x.GetCases()).Distinct().ToList();
    614630
    615631      foreach (var problem in problems) {
     632        if (double.IsNaN(problem.BestKnownQuality)) continue;
     633        rowNames.Add(problem.ToString());
    616634        var max = problem.IsMaximization();
    617         rowNames.Add(problem.ToString());
    618635        var absTargets = GetAbsoluteTargetsWorstToBest(problem);
    619636        if (targetsAreRelative) {
     
    625642        rowCount++;
    626643
    627         foreach (var alg in algorithmInstances) {
     644        foreach (var alg in groups) {
    628645          rowNames.Add(alg.Name);
    629           var runs = alg.GetRuns(problem).ToList();
     646          var runs = alg.GetTrials(problem).ToList();
    630647          if (runs.Count == 0) {
    631648            matrix[rowCount, columns - 1] = "N/A";
     
    635652          var result = default(ErtCalculationResult);
    636653          for (var i = 0; i < absTargets.Length; i++) {
    637             result = ExpectedRuntimeHelper.CalculateErt(runs, tableName, absTargets[i], max);
     654            result = ExpectedRuntimeHelper.CalculateErt(runs.Select(x => x.ToTuples()), absTargets[i], max);
    638655            matrix[rowCount, i] = result.ToString();
    639656          }
     
    658675
    659676      if (budgets == null) GenerateDefaultBudgets(table);
    660 
    661       var algInstances = GroupRuns().ToList();
    662       if (algInstances.Count == 0) return;
     677     
     678      if (groups.Count == 0) return;
    663679
    664680      var colorCount = 0;
    665681      var lineStyleCount = 0;
    666682     
    667       foreach (var alg in algInstances) {
     683      foreach (var alg in groups) {
    668684        var hits = new Dictionary<string, SortedList<double, double>>();
    669685
    670         foreach (var problem in alg.GetProblemInstances()) {
    671           foreach (var run in alg.GetRuns(problem)) {
    672             var resultsTable = (IndexedDataTable<double>)run.Results[table];
    673 
    674             CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), problem, alg.Name);
     686        foreach (var problem in alg.GetCases()) {
     687          foreach (var run in alg.GetTrials(problem)) {
     688            CalculateHitsForEachBudget(hits, run, problem, alg.Name);
    675689          }
    676690        }
     
    715729    }
    716730
    717     private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, ProblemInstance problem, string groupName) {
     731    private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, ConvergenceGraph cgraph, ProblemInstance problem, string groupName) {
    718732      var max = problem.IsMaximization();
    719733      var prevIndex = 0;
    720734      foreach (var b in budgets) {
    721735        var key = groupName + "-" + b;
    722         var index = row.Values.BinarySearch(prevIndex, row.Values.Count - prevIndex, Tuple.Create(b, 0.0), new CostComparer());
     736        var index = cgraph.BinarySearch(prevIndex, cgraph.Points - prevIndex, new ConvergenceGraphPoint(b, 0.0), new CostComparer());
    723737        if (index < 0) {
    724738          index = ~index;
    725           if (index >= row.Values.Count) break; // the run wasn't long enough to use up budget b (or any subsequent larger one)
     739          if (index >= cgraph.Points) break; // the run wasn't long enough to use up budget b (or any subsequent larger one)
    726740        }
    727741        if (!hits.ContainsKey(key)) hits.Add(key, new SortedList<double, double>());
    728         var v = row.Values[index];
    729         var relTgt = CalculateRelativeDifference(max, problem.BestKnownQuality, v.Item2) + 1;
     742        var v = cgraph[index];
     743        var relTgt = CalculateRelativeDifference(max, problem.BestKnownQuality, v.Quality) + 1;
    730744        if (hits[key].ContainsKey(relTgt))
    731745          hits[key][relTgt]++;
     
    874888        var pd = new ProblemInstance(run);
    875889        pd = problems.Single(x => x.Equals(pd));
     890        if (double.IsNaN(pd.BestKnownQuality)) continue;
     891
    876892        var max = pd.IsMaximization();
    877893        var absTargets = GetAbsoluteTargetsWorstToBest(pd);
     894        var cgraph = new ConvergenceGraph(resultsTable, max);
    878895
    879896        var prevIndex = 0;
    880897        for (var i = 0; i < absTargets.Length; i++) {
    881898          var absTarget = absTargets[i];
    882           var index = values.BinarySearch(prevIndex, values.Count - prevIndex, Tuple.Create(0.0, absTarget), new TargetComparer(max));
     899          var index = cgraph.BinarySearch(prevIndex, values.Count - prevIndex, new ConvergenceGraphPoint(0.0, absTarget), new TargetComparer(max));
    883900          if (index < 0) {
    884901            index = ~index;
     
    977994        var pd = new ProblemInstance(run);
    978995        pd = problems.Single(x => x.Equals(pd));
    979 
     996        var max = pd.IsMaximization();
     997        var cgraph = new ConvergenceGraph(resultsTable, max);
    980998        var prevIndex = 0;
    981999        foreach (var b in budgets) {
    982           var index = values.BinarySearch(prevIndex, values.Count - prevIndex, Tuple.Create(b, 0.0), new CostComparer());
     1000          var index = cgraph.BinarySearch(prevIndex, values.Count - prevIndex, new ConvergenceGraphPoint(b, 0.0), new CostComparer());
    9831001          if (index < 0) {
    9841002            index = ~index;
     
    10111029    private Dictionary<ProblemInstance, double> CalculateBestTargetPerProblemInstance(string table) {
    10121030      if (table == null) table = string.Empty;
    1013       return (from r in Content
    1014               where r.Visible
    1015               let pd = new ProblemInstance(r)
    1016               let target = r.Parameters.ContainsKey("BestKnownQuality")
    1017                            && r.Parameters["BestKnownQuality"] is DoubleValue
    1018                            && !double.IsNaN(((DoubleValue)r.Parameters["BestKnownQuality"]).Value)
    1019                 ? ((DoubleValue)r.Parameters["BestKnownQuality"]).Value
    1020                 : (r.Results.ContainsKey(table) && r.Results[table] is IndexedDataTable<double>
    1021                   ? ((IndexedDataTable<double>)r.Results[table]).Rows.First().Values.Last().Item2
    1022                   : double.NaN)
    1023               group target by pd into g
    1024               select new { Problem = g.Key, Target = g.Any(x => !double.IsNaN(x)) ? (g.Key.IsMaximization() ? g.Max() : g.Where(x => !double.IsNaN(x)).Min()) : double.NaN })
    1025         .ToDictionary(x => x.Problem, x => x.Target);
     1031      var dict = new Dictionary<ProblemInstance, double>();
     1032      foreach (var k in from r in Content where r.Visible let pd = new ProblemInstance(r) group r by pd into g select new { Problem = g.Key, Runs = g.ToList() }) {
     1033        var pd = new ProblemInstance(k.Runs[0]);
     1034        var values = GetQualityValues(k.Runs, table).ToList();
     1035        var target = double.NaN;
     1036        if (values.Count > 0) {
     1037          target = pd.IsMaximization() ? values.Max() : values.Min();
     1038        }
     1039        dict[pd] = target;
     1040      }
     1041      return dict;
     1042    }
     1043
     1044    private IEnumerable<double> GetQualityValues(List<IRun> runs, string table) {
     1045      foreach (var r in runs) {
     1046        IItem item;
     1047        if (r.Parameters.TryGetValue("BestKnownQuality", out item)) {
     1048          var dval = item as DoubleValue;
     1049          if (dval != null && !double.IsNaN(dval.Value))
     1050            yield return dval.Value;
     1051        }
     1052        if (r.Results.TryGetValue(table, out item)) {
     1053          var dt = item as IndexedDataTable<double>;
     1054          if (dt != null) {
     1055            var last = dt.Rows.First().Values.Last().Item2;
     1056            if (!double.IsNaN(last))
     1057              yield return last;
     1058          }
     1059        }
     1060      }
    10261061    }
    10271062
     
    10331068      SuspendRepaint();
    10341069      try {
     1070        GroupRuns();
    10351071        UpdateResultsByTarget();
    10361072        UpdateResultsByCost();
     
    11481184      }
    11491185
    1150       private Dictionary<ProblemInstance, List<IRun>> performanceData;
    1151 
    1152       public int GetNumberOfProblemInstances() {
     1186      private Dictionary<ProblemInstance, List<ConvergenceGraph>> performanceData;
     1187
     1188      public int GetNumberOfCases() {
    11531189        return performanceData.Count;
    11541190      }
    11551191
    1156       public IEnumerable<ProblemInstance> GetProblemInstances() {
     1192      public IEnumerable<ProblemInstance> GetCases() {
    11571193        return performanceData.Keys;
    11581194      }
    11591195
    1160       public int GetNumberOfRuns(ProblemInstance p) {
     1196      public int GetNumberOfTrials(ProblemInstance p) {
    11611197        if (p == ProblemInstance.MatchAll) return performanceData.Select(x => x.Value.Count).Sum();
    1162         List<IRun> runs;
    1163         if (performanceData.TryGetValue(p, out runs))
    1164           return runs.Count;
     1198        List<ConvergenceGraph> trials;
     1199        if (performanceData.TryGetValue(p, out trials))
     1200          return trials.Count;
    11651201
    11661202        return 0;
    11671203      }
    11681204
    1169       public IEnumerable<IRun> GetRuns(ProblemInstance p) {
     1205      public IEnumerable<ConvergenceGraph> GetTrials(ProblemInstance p) {
    11701206        if (p == ProblemInstance.MatchAll) return performanceData.SelectMany(x => x.Value);
    1171         List<IRun> runs;
    1172         if (performanceData.TryGetValue(p, out runs))
    1173           return runs;
    1174 
    1175         return Enumerable.Empty<IRun>();
    1176       }
    1177 
    1178       public AlgorithmInstance(string name, IEnumerable<IRun> runs, IEnumerable<ProblemInstance> problems) {
     1207        List<ConvergenceGraph> trials;
     1208        if (performanceData.TryGetValue(p, out trials))
     1209          return trials;
     1210
     1211        return Enumerable.Empty<ConvergenceGraph>();
     1212      }
     1213
     1214      public AlgorithmInstance(string name, IEnumerable<IGrouping<ProblemInstance, IndexedDataTable<double>>> trials) {
    11791215        this.name = name;
    11801216
    1181         var pDict = problems.ToDictionary(r => r, _ => new List<IRun>());
    1182         foreach (var y in runs) {
    1183           var pd = new ProblemInstance(y);
    1184           List<IRun> l;
    1185           if (pDict.TryGetValue(pd, out l)) l.Add(y);
    1186         }
    1187         performanceData = pDict.Where(x => x.Value.Count > 0).ToDictionary(x => x.Key, x => x.Value);
     1217        performanceData = new Dictionary<ProblemInstance, List<ConvergenceGraph>>();
     1218        foreach (var t in trials) {
     1219          if (double.IsNaN(t.Key.BestKnownQuality)) continue;
     1220          var max = t.Key.IsMaximization();
     1221          performanceData[t.Key] = t.Select(c => new ConvergenceGraph(c, max)).ToList();
     1222        }
    11881223      }
    11891224
     
    13871422    }
    13881423
    1389     private class CostComparer : Comparer<Tuple<double, double>> {
    1390       public override int Compare(Tuple<double, double> x, Tuple<double, double> y) {
    1391         return x.Item1.CompareTo(y.Item1);
    1392       }
    1393     }
    1394 
    1395     private class TargetComparer : Comparer<Tuple<double, double>> {
     1424    private class CostComparer : Comparer<ConvergenceGraphPoint> {
     1425      public override int Compare(ConvergenceGraphPoint x, ConvergenceGraphPoint y) {
     1426        return x.Runlength.CompareTo(y.Runlength);
     1427      }
     1428    }
     1429
     1430    private class TargetComparer : Comparer<ConvergenceGraphPoint> {
    13961431      public bool Maximization { get; private set; }
    13971432      public TargetComparer(bool maximization) {
     
    13991434      }
    14001435
    1401       public override int Compare(Tuple<double, double> x, Tuple<double, double> y) {
    1402         return Maximization ? x.Item2.CompareTo(y.Item2) : y.Item2.CompareTo(x.Item2);
     1436      public override int Compare(ConvergenceGraphPoint x, ConvergenceGraphPoint y) {
     1437        return Maximization ? x.Quality.CompareTo(y.Quality) : y.Quality.CompareTo(x.Quality);
     1438      }
     1439    }
     1440
     1441    private class ConvergenceGraph : IEnumerable<ConvergenceGraphPoint> {
     1442      private List<ConvergenceGraphPoint> data;
     1443      private bool maximization;
     1444      private string xAxisLabel;
     1445
     1446      public string XAxisName { get { return xAxisLabel; } }
     1447
     1448      public int Points {
     1449        get { return data.Count; }
     1450      }
     1451
     1452      public double TotalRunlength {
     1453        get { return data.Last().Runlength; }
     1454      }
     1455
     1456      public double BestQuality {
     1457        get { return data.Last().Quality; }
     1458      }
     1459
     1460      public double QualityAt(double runlength) {
     1461        var point = data.SkipWhile(x => x.Runlength < runlength).FirstOrDefault();
     1462        if (point == null) return double.NaN;
     1463        return point.Quality;
     1464      }
     1465
     1466      public double RunlengthFor(double quality) {
     1467        var point = (maximization ? data.SkipWhile(x => x.Quality < quality) : data.SkipWhile(x => x.Quality > quality)).FirstOrDefault();
     1468        if (point == null) return double.NaN;
     1469        return point.Runlength;
     1470      }
     1471
     1472      public ConvergenceGraphPoint this[int point] {
     1473        get { return data[point]; }
     1474      }
     1475
     1476      public ConvergenceGraph(IndexedDataTable<double> table, bool maximization) {
     1477        data = table.Rows.First().Values.Select(x => new ConvergenceGraphPoint() { Runlength = x.Item1, Quality = x.Item2 }).ToList();
     1478        xAxisLabel = table.VisualProperties.XAxisTitle;
     1479        this.maximization = maximization;
     1480      }
     1481
     1482      public IEnumerator<ConvergenceGraphPoint> GetEnumerator() {
     1483        return data.GetEnumerator();
     1484      }
     1485
     1486      IEnumerator IEnumerable.GetEnumerator() {
     1487        return data.GetEnumerator();
     1488      }
     1489
     1490      public int BinarySearch(ConvergenceGraphPoint item, IComparer<ConvergenceGraphPoint> comparer) {
     1491        return data.BinarySearch(item, comparer);
     1492      }
     1493
     1494      public int BinarySearch(int index, int count, ConvergenceGraphPoint point, IComparer<ConvergenceGraphPoint> comparer) {
     1495        return data.BinarySearch(index, count, point, comparer);
     1496      }
     1497
     1498      public IEnumerable<Tuple<double, double>> ToTuples() {
     1499        return data.Select(x => Tuple.Create(x.Runlength, x.Quality));
     1500      }
     1501    }
     1502
     1503    private class ConvergenceGraphPoint {
     1504      public double Runlength { get; set; }
     1505      public double Quality { get; set; }
     1506
     1507      public ConvergenceGraphPoint() {
     1508
     1509      }
     1510      public ConvergenceGraphPoint(double runlength, double quality) {
     1511        Runlength = runlength;
     1512        Quality = quality;
    14031513      }
    14041514    }
Note: See TracChangeset for help on using the changeset viewer.