Changeset 14775

Timestamp:

03/22/17 16:45:54 (8 years ago)

Author:

abeham

Message:

#2634: fixed bugs with respect to total number of runs

• some further restructuring to increase readability
• fixing numerical instability issues

Location:

trunk/sources

Files:

• HeuristicLab.Analysis/3.3/QualityAnalysis/ExpectedRuntimeHelper.cs (modified) (1 diff)
• HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs (modified) (17 diffs)

trunk/sources/HeuristicLab.Analysis/3.3/QualityAnalysis/ExpectedRuntimeHelper.cs

@@ -30 +30 @@
       if (successful.Count > 0) {
         var succAvg = successful.Average();
-        var succDev = successful.StandardDeviation();
+        var succDev = successful.StandardDeviation() + 1e-7;
         successful.RemoveAll(x => x < succAvg - 2 * succDev);
         unsuccessful.RemoveAll(x => x < succAvg - 2 * succDev);

trunk/sources/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs

@@ -1 +1 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -42 +42 @@
     private List<Series> invisibleTargetSeries;
 
-    private const string AllRuns = "All Runs";
+    private const string AllInstances = "All Instances";
 
     private static readonly Color[] colors = new[] {
@@ -223 +223 @@
       var groupings = Content.ParameterNames.OrderBy(x => x).ToArray();
       groupComboBox.Items.Clear();
-      groupComboBox.Items.Add(AllRuns);
+      groupComboBox.Items.Add(AllInstances);
       groupComboBox.Items.AddRange(groupings);
       if (selectedGroupItem != null && groupComboBox.Items.Contains(selectedGroupItem)) {
@@ -241 +241 @@
 
       var selectedProblemItem = (ProblemInstance)problemComboBox.SelectedItem;
+      problemComboBox.DataSource = null;
+      problemComboBox.Items.Clear();
       problems.Clear();
       problems.Add(ProblemInstance.MatchAll);
+      problemComboBox.DataSource = new BindingSource() { DataSource = problems };
+      problemComboBox.DataBindings.DefaultDataSourceUpdateMode = DataSourceUpdateMode.OnPropertyChanged;
       if (problems[0].Equals(selectedProblemItem)) problemComboBox.SelectedItem = problems[0];
       foreach (var p in problemDict.ToList()) {
@@ -282 +286 @@
     }
 
-    private Dictionary<string, Dictionary<ProblemInstance, List<IRun>>> GroupRuns() {
-      var groupedRuns = new Dictionary<string, Dictionary<ProblemInstance, List<IRun>>>();
-
+    private IEnumerable<AlgorithmInstance> GroupRuns() {
       var table = (string)dataTableComboBox.SelectedItem;
-      if (string.IsNullOrEmpty(table)) return groupedRuns;
+      if (string.IsNullOrEmpty(table)) yield break;
 
       var selectedGroup = (string)groupComboBox.SelectedItem;
-      if (string.IsNullOrEmpty(selectedGroup)) return groupedRuns;
+      if (string.IsNullOrEmpty(selectedGroup)) yield break;
 
       var selectedProblem = (ProblemInstance)problemComboBox.SelectedItem;
-      if (selectedProblem == null) return groupedRuns;
+      if (selectedProblem == null) yield break;
       
       foreach (var x in (from r in Content
-                         where (selectedGroup == AllRuns || r.Parameters.ContainsKey(selectedGroup))
+                         where (selectedGroup == AllInstances || r.Parameters.ContainsKey(selectedGroup))
                            && selectedProblem.Match(r)
                            && r.Results.ContainsKey(table)
                            && r.Visible
-                         let key = selectedGroup == AllRuns ? AllRuns : r.Parameters[selectedGroup].ToString()
+                         let key = selectedGroup == AllInstances ? AllInstances : r.Parameters[selectedGroup].ToString()
                          group r by key into g
-                         select Tuple.Create(g.Key, g.ToList()))) {
-        var pDict = problems.ToDictionary(r => r, _ => new List<IRun>());
-        foreach (var y in x.Item2) {
-          var pd = new ProblemInstance(y);
-          List<IRun> l;
-          if (pDict.TryGetValue(pd, out l)) l.Add(y);
-        }
-        groupedRuns[x.Item1] = pDict.Where(a => a.Value.Count > 0).ToDictionary(a => a.Key, a => a.Value);
-      }
-
-      return groupedRuns;
+                         select new AlgorithmInstance(g.Key, g, problems))) {
+        yield return x;
+      }
     }
 
@@ -326 +320 @@
       if (targets == null) GenerateDefaultTargets();
 
-      var groupedRuns = GroupRuns();
-      if (groupedRuns.Count == 0) return;
+      var algInstances = GroupRuns().ToList();
+      if (algInstances.Count == 0) return;
 
       var xAxisTitles = new HashSet<string>();
@@ -337 +331 @@
       // how many targets have been left open at the point when the run ended
       var misses = new Dictionary<string, SortedList<double, int>>();
+      var totalRuns = new Dictionary<string, int>();
 
       var aggregate = aggregateTargetsCheckBox.Checked;
       double minEff = double.MaxValue, maxEff = double.MinValue;
-      var noRuns = 0;
-      foreach (var group in groupedRuns) {
+      foreach (var alg in algInstances) {
+        var noRuns = 0;
         SortedList<double, int> epdfHits = null, epdfMisses = null;
         if (aggregate) {
-          hits[group.Key] = epdfHits = new SortedList<double, int>();
-          misses[group.Key] = epdfMisses = new SortedList<double, int>();
-        }
-        foreach (var problem in group.Value) {
-          var max = problem.Key.IsMaximization();
-          var absTargets = GetAbsoluteTargets(problem.Key).ToArray();
-          foreach (var run in problem.Value) {
+          hits[alg.Name] = epdfHits = new SortedList<double, int>();
+          misses[alg.Name] = epdfMisses = new SortedList<double, int>();
+        }
+        foreach (var problem in alg.GetProblemInstances()) {
+          var max = problem.IsMaximization();
+          var absTargets = GetAbsoluteTargets(problem).ToArray();
+          foreach (var run in alg.GetRuns(problem)) {
             noRuns++;
             var resultsTable = (IndexedDataTable<double>)run.Results[table];
@@ -361 +356 @@
               var e = efforts[idx];
               if (!aggregate) {
-                var key = group.Key + "@" + (targetsAreRelative
-                  ? (targets[idx] * 100).ToString(CultureInfo.CurrentCulture.NumberFormat) + "%"
-                  : targets[idx].ToString(CultureInfo.CurrentCulture.NumberFormat));
+                var key = alg.Name + "@" + (targetsAreRelative
+                            ? (targets[idx] * 100).ToString(CultureInfo.CurrentCulture.NumberFormat) + "%"
+                            : targets[idx].ToString(CultureInfo.CurrentCulture.NumberFormat));
                 if (!hits.TryGetValue(key, out epdfHits))
                   hits[key] = epdfHits = new SortedList<double, int>();
                 if (!misses.TryGetValue(key, out epdfMisses))
                   misses[key] = epdfMisses = new SortedList<double, int>();
-              }
+                totalRuns[key] = noRuns;
+              };
               var list = e.Item1 ? epdfHits : epdfMisses;
               int v;
@@ -377 +373 @@
           }
         }
+        if (aggregate) totalRuns[alg.Name] = noRuns;
       }
 
       UpdateTargetChartAxisXBounds(minEff, maxEff);
 
-      DrawTargetsEcdf(hits, misses, noRuns);
+      DrawTargetsEcdf(hits, misses, totalRuns);
 
       if (targets.Length == 1) {
@@ -392 +389 @@
       targetChart.ChartAreas[0].CursorY.Interval = 0.05;
 
-      UpdateErtTables(groupedRuns);
-    }
-
-    private void DrawTargetsEcdf(Dictionary<string, SortedList<double, int>> hits, Dictionary<string, SortedList<double, int>> misses, int noRuns) {
+      UpdateErtTables(algInstances);
+    }
+
+    private void DrawTargetsEcdf(Dictionary<string, SortedList<double, int>> hits, Dictionary<string, SortedList<double, int>> misses, Dictionary<string, int> noRuns) {
       var colorCount = 0;
       var lineStyleCount = 0;
@@ -418 +415 @@
         var iter = misses[list.Key].GetEnumerator();
         var moreMisses = iter.MoveNext();
-        var totalTargets = noRuns;
+        var totalTargets = noRuns[list.Key];
         if (aggregateTargetsCheckBox.Checked) totalTargets *= targets.Length;
         var movingTargets = totalTargets;
@@ -594 +591 @@
     }
 
-    private void UpdateErtTables(Dictionary<string, Dictionary<ProblemInstance, List<IRun>>> groupedRuns) {
+    private void UpdateErtTables(List<AlgorithmInstance> algorithmInstances) {
       ertTableView.Content = null;
       var columns = 1 + targets.Length + 1;
-      var matrix = new string[groupedRuns.Count * groupedRuns.Max(x => x.Value.Count) + groupedRuns.Max(x => x.Value.Count), columns];
+      var matrix = new string[algorithmInstances.Count * algorithmInstances.Max(x => x.GetNumberOfProblemInstances()) + algorithmInstances.Max(x => x.GetNumberOfProblemInstances()), columns];
       var rowCount = 0;
 
@@ -603 +600 @@
       if (string.IsNullOrEmpty(tableName)) return;
       
-      var problems = groupedRuns.SelectMany(x => x.Value.Keys).Distinct().ToList();
+      var problems = algorithmInstances.SelectMany(x => x.GetProblemInstances()).Distinct().ToList();
 
       foreach (var problem in problems) {
@@ -615 +612 @@
         rowCount++;
 
-        foreach (var group in groupedRuns) {
-          matrix[rowCount, 0] = group.Key;
-          if (!group.Value.ContainsKey(problem)) {
+        foreach (var alg in algorithmInstances) {
+          matrix[rowCount, 0] = alg.Name;
+          var runs = alg.GetRuns(problem).ToList();
+          if (runs.Count == 0) {
             matrix[rowCount, columns - 1] = "N/A";
             rowCount++;
             continue;
           }
-          var runs = group.Value[problem];
           var result = default(ErtCalculationResult);
           for (var i = 0; i < absTargets.Length; i++) {
@@ -648 +645 @@
       if (budgets == null) GenerateDefaultBudgets(table);
 
-      var groupedRuns = GroupRuns();
-      if (groupedRuns.Count == 0) return;
+      var algInstances = GroupRuns().ToList();
+      if (algInstances.Count == 0) return;
 
       var colorCount = 0;
       var lineStyleCount = 0;
       
-      foreach (var group in groupedRuns) {
+      foreach (var alg in algInstances) {
         var hits = new Dictionary<string, SortedList<double, double>>();
 
-        foreach (var problem in group.Value) {
-          foreach (var run in problem.Value) {
+        foreach (var problem in alg.GetProblemInstances()) {
+          foreach (var run in alg.GetRuns(problem)) {
             var resultsTable = (IndexedDataTable<double>)run.Results[table];
 
             if (aggregateBudgetsCheckBox.Checked) {
-              CalculateHitsForAllBudgets(hits, resultsTable.Rows.First(), group.Value.Count, problem.Key, group.Key, problem.Value.Count);
+              CalculateHitsForAllBudgets(hits, resultsTable.Rows.First(), alg.GetNumberOfProblemInstances(), problem, alg.Name, alg.GetNumberOfRuns(problem));
             } else {
-              CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), group.Value.Count, problem.Key, group.Key, problem.Value.Count);
+              CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), alg.GetNumberOfProblemInstances(), problem, alg.Name, alg.GetNumberOfRuns(problem));
             }
           }
@@ -697 +694 @@
 
     private void GenerateDefaultBudgets(string table) {
-      var runs = GroupRuns().SelectMany(x => x.Value.Values).SelectMany(x => x).ToList();
+      var runs = Content;
       var min = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Min()).Min();
       var max = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Max()).Max();
@@ -1172 +1169 @@
     }
 
+    private class AlgorithmInstance : INotifyPropertyChanged {
+      private string name;
+      public string Name {
+        get { return name; }
+        set {
+          if (name == value) return;
+          name = value;
+          OnPropertyChanged("Name");
+        }
+      }
+
+      private Dictionary<ProblemInstance, List<IRun>> performanceData;
+
+      public int GetNumberOfProblemInstances() {
+        return performanceData.Count;
+      }
+
+      public IEnumerable<ProblemInstance> GetProblemInstances() {
+        return performanceData.Keys;
+      }
+
+      public int GetNumberOfRuns(ProblemInstance p) {
+        if (p == ProblemInstance.MatchAll) return performanceData.Select(x => x.Value.Count).Sum();
+        List<IRun> runs;
+        if (performanceData.TryGetValue(p, out runs))
+          return runs.Count;
+
+        return 0;
+      }
+
+      public IEnumerable<IRun> GetRuns(ProblemInstance p) {
+        if (p == ProblemInstance.MatchAll) return performanceData.SelectMany(x => x.Value);
+        List<IRun> runs;
+        if (performanceData.TryGetValue(p, out runs))
+          return runs;
+
+        return Enumerable.Empty<IRun>();
+      }
+
+      public AlgorithmInstance(string name, IEnumerable<IRun> runs, IEnumerable<ProblemInstance> problems) {
+        this.name = name;
+
+        var pDict = problems.ToDictionary(r => r, _ => new List<IRun>());
+        foreach (var y in runs) {
+          var pd = new ProblemInstance(y);
+          List<IRun> l;
+          if (pDict.TryGetValue(pd, out l)) l.Add(y);
+        }
+        performanceData = pDict.Where(x => x.Value.Count > 0).ToDictionary(x => x.Key, x => x.Value);
+      }
+
+      public override bool Equals(object obj) {
+        var other = obj as AlgorithmInstance;
+        if (other == null) return false;
+        return name == other.name;
+      }
+
+      public override int GetHashCode() {
+        return name.GetHashCode();
+      }
+
+      public event PropertyChangedEventHandler PropertyChanged;
+      protected virtual void OnPropertyChanged(string propertyName = null) {
+        var handler = PropertyChanged;
+        if (handler != null) handler(this, new PropertyChangedEventArgs(propertyName));
+      }
+    }
+
     private class ProblemInstance : INotifyPropertyChanged {
       private readonly bool matchAll;