Changeset 14775


Ignore:
Timestamp:
03/22/17 16:45:54 (17 months ago)
Author:
abeham
Message:

#2634: fixed bugs with respect to total number of runs

  • some further restructuring to increase readability
  • fixing numerical instability issues
Location:
trunk/sources
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • trunk/sources/HeuristicLab.Analysis/3.3/QualityAnalysis/ExpectedRuntimeHelper.cs

    r14102 r14775  
    3030      if (successful.Count > 0) {
    3131        var succAvg = successful.Average();
    32         var succDev = successful.StandardDeviation();
     32        var succDev = successful.StandardDeviation() + 1e-7;
    3333        successful.RemoveAll(x => x < succAvg - 2 * succDev);
    3434        unsuccessful.RemoveAll(x => x < succAvg - 2 * succDev);
  • trunk/sources/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs

    r14654 r14775  
    11#region License Information
    22/* HeuristicLab
    3  * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
     3 * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
    44 *
    55 * This file is part of HeuristicLab.
     
    4242    private List<Series> invisibleTargetSeries;
    4343
    44     private const string AllRuns = "All Runs";
     44    private const string AllInstances = "All Instances";
    4545
    4646    private static readonly Color[] colors = new[] {
     
    223223      var groupings = Content.ParameterNames.OrderBy(x => x).ToArray();
    224224      groupComboBox.Items.Clear();
    225       groupComboBox.Items.Add(AllRuns);
     225      groupComboBox.Items.Add(AllInstances);
    226226      groupComboBox.Items.AddRange(groupings);
    227227      if (selectedGroupItem != null && groupComboBox.Items.Contains(selectedGroupItem)) {
     
    241241
    242242      var selectedProblemItem = (ProblemInstance)problemComboBox.SelectedItem;
     243      problemComboBox.DataSource = null;
     244      problemComboBox.Items.Clear();
    243245      problems.Clear();
    244246      problems.Add(ProblemInstance.MatchAll);
     247      problemComboBox.DataSource = new BindingSource() { DataSource = problems };
     248      problemComboBox.DataBindings.DefaultDataSourceUpdateMode = DataSourceUpdateMode.OnPropertyChanged;
    245249      if (problems[0].Equals(selectedProblemItem)) problemComboBox.SelectedItem = problems[0];
    246250      foreach (var p in problemDict.ToList()) {
     
    282286    }
    283287
    284     private Dictionary<string, Dictionary<ProblemInstance, List<IRun>>> GroupRuns() {
    285       var groupedRuns = new Dictionary<string, Dictionary<ProblemInstance, List<IRun>>>();
    286 
     288    private IEnumerable<AlgorithmInstance> GroupRuns() {
    287289      var table = (string)dataTableComboBox.SelectedItem;
    288       if (string.IsNullOrEmpty(table)) return groupedRuns;
     290      if (string.IsNullOrEmpty(table)) yield break;
    289291
    290292      var selectedGroup = (string)groupComboBox.SelectedItem;
    291       if (string.IsNullOrEmpty(selectedGroup)) return groupedRuns;
     293      if (string.IsNullOrEmpty(selectedGroup)) yield break;
    292294
    293295      var selectedProblem = (ProblemInstance)problemComboBox.SelectedItem;
    294       if (selectedProblem == null) return groupedRuns;
     296      if (selectedProblem == null) yield break;
    295297     
    296298      foreach (var x in (from r in Content
    297                          where (selectedGroup == AllRuns || r.Parameters.ContainsKey(selectedGroup))
     299                         where (selectedGroup == AllInstances || r.Parameters.ContainsKey(selectedGroup))
    298300                           && selectedProblem.Match(r)
    299301                           && r.Results.ContainsKey(table)
    300302                           && r.Visible
    301                          let key = selectedGroup == AllRuns ? AllRuns : r.Parameters[selectedGroup].ToString()
     303                         let key = selectedGroup == AllInstances ? AllInstances : r.Parameters[selectedGroup].ToString()
    302304                         group r by key into g
    303                          select Tuple.Create(g.Key, g.ToList()))) {
    304         var pDict = problems.ToDictionary(r => r, _ => new List<IRun>());
    305         foreach (var y in x.Item2) {
    306           var pd = new ProblemInstance(y);
    307           List<IRun> l;
    308           if (pDict.TryGetValue(pd, out l)) l.Add(y);
    309         }
    310         groupedRuns[x.Item1] = pDict.Where(a => a.Value.Count > 0).ToDictionary(a => a.Key, a => a.Value);
    311       }
    312 
    313       return groupedRuns;
     305                         select new AlgorithmInstance(g.Key, g, problems))) {
     306        yield return x;
     307      }
    314308    }
    315309
     
    326320      if (targets == null) GenerateDefaultTargets();
    327321
    328       var groupedRuns = GroupRuns();
    329       if (groupedRuns.Count == 0) return;
     322      var algInstances = GroupRuns().ToList();
     323      if (algInstances.Count == 0) return;
    330324
    331325      var xAxisTitles = new HashSet<string>();
     
    337331      // how many targets have been left open at the point when the run ended
    338332      var misses = new Dictionary<string, SortedList<double, int>>();
     333      var totalRuns = new Dictionary<string, int>();
    339334
    340335      var aggregate = aggregateTargetsCheckBox.Checked;
    341336      double minEff = double.MaxValue, maxEff = double.MinValue;
    342       var noRuns = 0;
    343       foreach (var group in groupedRuns) {
     337      foreach (var alg in algInstances) {
     338        var noRuns = 0;
    344339        SortedList<double, int> epdfHits = null, epdfMisses = null;
    345340        if (aggregate) {
    346           hits[group.Key] = epdfHits = new SortedList<double, int>();
    347           misses[group.Key] = epdfMisses = new SortedList<double, int>();
    348         }
    349         foreach (var problem in group.Value) {
    350           var max = problem.Key.IsMaximization();
    351           var absTargets = GetAbsoluteTargets(problem.Key).ToArray();
    352           foreach (var run in problem.Value) {
     341          hits[alg.Name] = epdfHits = new SortedList<double, int>();
     342          misses[alg.Name] = epdfMisses = new SortedList<double, int>();
     343        }
     344        foreach (var problem in alg.GetProblemInstances()) {
     345          var max = problem.IsMaximization();
     346          var absTargets = GetAbsoluteTargets(problem).ToArray();
     347          foreach (var run in alg.GetRuns(problem)) {
    353348            noRuns++;
    354349            var resultsTable = (IndexedDataTable<double>)run.Results[table];
     
    361356              var e = efforts[idx];
    362357              if (!aggregate) {
    363                 var key = group.Key + "@" + (targetsAreRelative
    364                   ? (targets[idx] * 100).ToString(CultureInfo.CurrentCulture.NumberFormat) + "%"
    365                   : targets[idx].ToString(CultureInfo.CurrentCulture.NumberFormat));
     358                var key = alg.Name + "@" + (targetsAreRelative
     359                            ? (targets[idx] * 100).ToString(CultureInfo.CurrentCulture.NumberFormat) + "%"
     360                            : targets[idx].ToString(CultureInfo.CurrentCulture.NumberFormat));
    366361                if (!hits.TryGetValue(key, out epdfHits))
    367362                  hits[key] = epdfHits = new SortedList<double, int>();
    368363                if (!misses.TryGetValue(key, out epdfMisses))
    369364                  misses[key] = epdfMisses = new SortedList<double, int>();
    370               }
     365                totalRuns[key] = noRuns;
     366              };
    371367              var list = e.Item1 ? epdfHits : epdfMisses;
    372368              int v;
     
    377373          }
    378374        }
     375        if (aggregate) totalRuns[alg.Name] = noRuns;
    379376      }
    380377
    381378      UpdateTargetChartAxisXBounds(minEff, maxEff);
    382379
    383       DrawTargetsEcdf(hits, misses, noRuns);
     380      DrawTargetsEcdf(hits, misses, totalRuns);
    384381
    385382      if (targets.Length == 1) {
     
    392389      targetChart.ChartAreas[0].CursorY.Interval = 0.05;
    393390
    394       UpdateErtTables(groupedRuns);
    395     }
    396 
    397     private void DrawTargetsEcdf(Dictionary<string, SortedList<double, int>> hits, Dictionary<string, SortedList<double, int>> misses, int noRuns) {
     391      UpdateErtTables(algInstances);
     392    }
     393
     394    private void DrawTargetsEcdf(Dictionary<string, SortedList<double, int>> hits, Dictionary<string, SortedList<double, int>> misses, Dictionary<string, int> noRuns) {
    398395      var colorCount = 0;
    399396      var lineStyleCount = 0;
     
    418415        var iter = misses[list.Key].GetEnumerator();
    419416        var moreMisses = iter.MoveNext();
    420         var totalTargets = noRuns;
     417        var totalTargets = noRuns[list.Key];
    421418        if (aggregateTargetsCheckBox.Checked) totalTargets *= targets.Length;
    422419        var movingTargets = totalTargets;
     
    594591    }
    595592
    596     private void UpdateErtTables(Dictionary<string, Dictionary<ProblemInstance, List<IRun>>> groupedRuns) {
     593    private void UpdateErtTables(List<AlgorithmInstance> algorithmInstances) {
    597594      ertTableView.Content = null;
    598595      var columns = 1 + targets.Length + 1;
    599       var matrix = new string[groupedRuns.Count * groupedRuns.Max(x => x.Value.Count) + groupedRuns.Max(x => x.Value.Count), columns];
     596      var matrix = new string[algorithmInstances.Count * algorithmInstances.Max(x => x.GetNumberOfProblemInstances()) + algorithmInstances.Max(x => x.GetNumberOfProblemInstances()), columns];
    600597      var rowCount = 0;
    601598
     
    603600      if (string.IsNullOrEmpty(tableName)) return;
    604601     
    605       var problems = groupedRuns.SelectMany(x => x.Value.Keys).Distinct().ToList();
     602      var problems = algorithmInstances.SelectMany(x => x.GetProblemInstances()).Distinct().ToList();
    606603
    607604      foreach (var problem in problems) {
     
    615612        rowCount++;
    616613
    617         foreach (var group in groupedRuns) {
    618           matrix[rowCount, 0] = group.Key;
    619           if (!group.Value.ContainsKey(problem)) {
     614        foreach (var alg in algorithmInstances) {
     615          matrix[rowCount, 0] = alg.Name;
     616          var runs = alg.GetRuns(problem).ToList();
     617          if (runs.Count == 0) {
    620618            matrix[rowCount, columns - 1] = "N/A";
    621619            rowCount++;
    622620            continue;
    623621          }
    624           var runs = group.Value[problem];
    625622          var result = default(ErtCalculationResult);
    626623          for (var i = 0; i < absTargets.Length; i++) {
     
    648645      if (budgets == null) GenerateDefaultBudgets(table);
    649646
    650       var groupedRuns = GroupRuns();
    651       if (groupedRuns.Count == 0) return;
     647      var algInstances = GroupRuns().ToList();
     648      if (algInstances.Count == 0) return;
    652649
    653650      var colorCount = 0;
    654651      var lineStyleCount = 0;
    655652     
    656       foreach (var group in groupedRuns) {
     653      foreach (var alg in algInstances) {
    657654        var hits = new Dictionary<string, SortedList<double, double>>();
    658655
    659         foreach (var problem in group.Value) {
    660           foreach (var run in problem.Value) {
     656        foreach (var problem in alg.GetProblemInstances()) {
     657          foreach (var run in alg.GetRuns(problem)) {
    661658            var resultsTable = (IndexedDataTable<double>)run.Results[table];
    662659
    663660            if (aggregateBudgetsCheckBox.Checked) {
    664               CalculateHitsForAllBudgets(hits, resultsTable.Rows.First(), group.Value.Count, problem.Key, group.Key, problem.Value.Count);
     661              CalculateHitsForAllBudgets(hits, resultsTable.Rows.First(), alg.GetNumberOfProblemInstances(), problem, alg.Name, alg.GetNumberOfRuns(problem));
    665662            } else {
    666               CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), group.Value.Count, problem.Key, group.Key, problem.Value.Count);
     663              CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), alg.GetNumberOfProblemInstances(), problem, alg.Name, alg.GetNumberOfRuns(problem));
    667664            }
    668665          }
     
    697694
    698695    private void GenerateDefaultBudgets(string table) {
    699       var runs = GroupRuns().SelectMany(x => x.Value.Values).SelectMany(x => x).ToList();
     696      var runs = Content;
    700697      var min = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Min()).Min();
    701698      var max = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Max()).Max();
     
    11721169    }
    11731170
     1171    private class AlgorithmInstance : INotifyPropertyChanged {
     1172      private string name;
     1173      public string Name {
     1174        get { return name; }
     1175        set {
     1176          if (name == value) return;
     1177          name = value;
     1178          OnPropertyChanged("Name");
     1179        }
     1180      }
     1181
     1182      private Dictionary<ProblemInstance, List<IRun>> performanceData;
     1183
     1184      public int GetNumberOfProblemInstances() {
     1185        return performanceData.Count;
     1186      }
     1187
     1188      public IEnumerable<ProblemInstance> GetProblemInstances() {
     1189        return performanceData.Keys;
     1190      }
     1191
     1192      public int GetNumberOfRuns(ProblemInstance p) {
     1193        if (p == ProblemInstance.MatchAll) return performanceData.Select(x => x.Value.Count).Sum();
     1194        List<IRun> runs;
     1195        if (performanceData.TryGetValue(p, out runs))
     1196          return runs.Count;
     1197
     1198        return 0;
     1199      }
     1200
     1201      public IEnumerable<IRun> GetRuns(ProblemInstance p) {
     1202        if (p == ProblemInstance.MatchAll) return performanceData.SelectMany(x => x.Value);
     1203        List<IRun> runs;
     1204        if (performanceData.TryGetValue(p, out runs))
     1205          return runs;
     1206
     1207        return Enumerable.Empty<IRun>();
     1208      }
     1209
     1210      public AlgorithmInstance(string name, IEnumerable<IRun> runs, IEnumerable<ProblemInstance> problems) {
     1211        this.name = name;
     1212
     1213        var pDict = problems.ToDictionary(r => r, _ => new List<IRun>());
     1214        foreach (var y in runs) {
     1215          var pd = new ProblemInstance(y);
     1216          List<IRun> l;
     1217          if (pDict.TryGetValue(pd, out l)) l.Add(y);
     1218        }
     1219        performanceData = pDict.Where(x => x.Value.Count > 0).ToDictionary(x => x.Key, x => x.Value);
     1220      }
     1221
     1222      public override bool Equals(object obj) {
     1223        var other = obj as AlgorithmInstance;
     1224        if (other == null) return false;
     1225        return name == other.name;
     1226      }
     1227
     1228      public override int GetHashCode() {
     1229        return name.GetHashCode();
     1230      }
     1231
     1232      public event PropertyChangedEventHandler PropertyChanged;
     1233      protected virtual void OnPropertyChanged(string propertyName = null) {
     1234        var handler = PropertyChanged;
     1235        if (handler != null) handler(this, new PropertyChangedEventArgs(propertyName));
     1236      }
     1237    }
     1238
    11741239    private class ProblemInstance : INotifyPropertyChanged {
    11751240      private readonly bool matchAll;
Note: See TracChangeset for help on using the changeset viewer.