Changeset 12864 for branches

Timestamp:

08/16/15 00:53:19 (9 years ago)

Author:

abeham

Message:

#2431:

• fixed bug in iterated algorithm
• changed RLD view to use % deviation as targets
• RLD view now always groups by problem

Location:

branches/PerformanceComparison

Files:

• HeuristicLab.Analysis/3.3/Optimizers/IteratedAlgorithm.cs (modified) (3 diffs)
• HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs (modified) (22 diffs)

branches/PerformanceComparison/HeuristicLab.Analysis/3.3/Optimizers/IteratedAlgorithm.cs

@@ -105 +105 @@
     }
 
-    [Storable]
     private IAlgorithm algorithm;
     public IAlgorithm Algorithm {
@@ -157 +156 @@
       perClockAnalyzer = cloner.Clone(original.perClockAnalyzer);
       perEvaluationsAnalyzer = cloner.Clone(original.perEvaluationsAnalyzer);
-
+      algorithm = cloner.Clone(original.algorithm);
       RegisterEventHandlers();
     }
@@ -183 +182 @@
     [StorableHook(HookType.AfterDeserialization)]
     private void AfterDeserialization() {
+      algorithm = AlgorithmParameter.Value;
       RegisterEventHandlers();
     }

branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs

@@ -220 +220 @@
       }
 
-      var problemTypesDifferent = problems.Select(x => x.ProblemType).Distinct().Count() > 1;
-      var problemNamesDifferent = problems.Select(x => x.ProblemName).Distinct().Count() > 1;
-      var evaluatorDifferent = problems.Select(x => x.Evaluator).Distinct().Count() > 1;
+      var problemTypesDifferent = problems.Select(x => x.ProblemType).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
+      var problemNamesDifferent = problems.Select(x => x.ProblemName).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
+      var evaluatorDifferent = problems.Select(x => x.Evaluator).Where(x => !string.IsNullOrEmpty(x)).Distinct().Count() > 1;
       var allEqual = !problemTypesDifferent && !problemNamesDifferent && !evaluatorDifferent;
 
       var selectedProblemItem = (ProblemDescription)problemComboBox.SelectedItem;
       problemComboBox.Items.Clear();
+      problemComboBox.Items.Add(ProblemDescription.MatchAll);
+      if (selectedProblemItem == null || selectedProblemItem == ProblemDescription.MatchAll)
+        problemComboBox.SelectedIndex = 0;
       foreach (var prob in problems.OrderBy(x => x.ToString()).ToList()) {
         prob.DisplayProblemType = problemTypesDifferent;
@@ -263 +266 @@
     }
 
-    private List<Tuple<string, List<IRun>>> GroupRuns() {
-      var groupedRuns = new List<Tuple<string, List<IRun>>>();
+    private Dictionary<string, Dictionary<string, Tuple<double, List<IRun>>>> GroupRuns() {
+      var groupedRuns = new Dictionary<string, Dictionary<string, Tuple<double, List<IRun>>>>();
 
       var table = (string)dataTableComboBox.SelectedItem;
@@ -273 +276 @@
 
       var selectedProblem = (ProblemDescription)problemComboBox.SelectedItem;
-      if (selectedProblem == null) selectedProblem = ProblemDescription.MatchAll;
-
-      if (selectedGroup == AllRuns)
-        groupedRuns = new List<Tuple<string, List<IRun>>> {
-          Tuple.Create(AllRuns, Content.Where(r => selectedProblem.Match(r) && r.Results.ContainsKey(table) && r.Visible).ToList())
-        };
-      else
-        groupedRuns = (from r in Content
-                       where r.Parameters.ContainsKey(selectedGroup)
-                             && selectedProblem.Match(r)
-                             && r.Results.ContainsKey(table)
-                             && r.Visible
-                       group r by r.Parameters[selectedGroup].ToString()
-                         into g
-                         select Tuple.Create(g.Key, g.ToList())).ToList();
+      if (selectedProblem == null) return groupedRuns;
+
+      var maximization = IsMaximization();
+
+      var targetsPerProblem = (from r in Content
+                               let pd = new ProblemDescription(r).ToString()
+                               let target = r.Parameters.ContainsKey("BestKnownQuality")
+                                && r.Parameters["BestKnownQuality"] is DoubleValue
+                                  ? ((DoubleValue)r.Parameters["BestKnownQuality"]).Value
+                                  : ((IndexedDataTable<double>)r.Results[table]).Rows.First().Values.Last().Item2
+                               group target by pd into g
+                               select new { Problem = g.Key, Target = maximization ? g.Max() : g.Min() })
+                              .ToDictionary(x => x.Problem, x => x.Target);
+
+      foreach (var x in (from r in Content
+                         where selectedGroup == AllRuns || r.Parameters.ContainsKey(selectedGroup)
+                           && selectedProblem.Match(r)
+                           && r.Results.ContainsKey(table)
+                           && r.Visible
+                         group r by selectedGroup == AllRuns ? AllRuns : r.Parameters[selectedGroup].ToString() into g
+                         select Tuple.Create(g.Key, g.ToList()))) {
+        var pDict = new Dictionary<string, Tuple<double, List<IRun>>>();
+        foreach (var y in (from r in x.Item2
+                           let pd = new ProblemDescription(r).ToString()
+                           group r by pd into g
+                           select Tuple.Create(g.Key, g.ToList()))) {
+          pDict[y.Item1] = Tuple.Create(targetsPerProblem[y.Item1], y.Item2);
+        }
+        groupedRuns[x.Item1] = pDict;
+      }
 
       return groupedRuns;
@@ -313 +331 @@
       if (string.IsNullOrEmpty(table)) return;
 
-      if (targets == null) GenerateDefaultTargets(table);
+      if (targets == null) GenerateDefaultTargets();
 
       var groupedRuns = GroupRuns();
@@ -327 +345 @@
         var maxLength = 0.0;
 
-        foreach (var run in group.Item2) {
-          var resultsTable = (IndexedDataTable<double>)run.Results[table];
-          xAxisTitles.Add(resultsTable.VisualProperties.XAxisTitle);
-
-          if (eachOrAllTargetCheckBox.Checked) {
-            CalculateHitsForEachTarget(hits, resultsTable.Rows.First(), maximization, group.Item2.Count, group.Item1);
-          } else {
-            maxLength = CalculateHitsForAllTargets(hits, resultsTable.Rows.First(), maximization, group.Item2.Count, group.Item1);
+        foreach (var problem in group.Value) {
+          foreach (var run in problem.Value.Item2) {
+            var resultsTable = (IndexedDataTable<double>)run.Results[table];
+            xAxisTitles.Add(resultsTable.VisualProperties.XAxisTitle);
+
+            if (eachOrAllTargetCheckBox.Checked) {
+              CalculateHitsForEachTarget(hits, resultsTable.Rows.First(), maximization, group.Key, group.Value.Count, problem.Value.Item1, problem.Value.Item2.Count);
+            } else {
+              maxLength = CalculateHitsForAllTargets(hits, resultsTable.Rows.First(), maximization, group.Key, group.Value.Count, problem.Value.Item1, problem.Value.Item2.Count);
+            }
           }
         }
@@ -369 +389 @@
     }
 
-    private void GenerateDefaultTargets(string table) {
-      var runs = GroupRuns().SelectMany(x => x.Item2).ToList();
-      var worst = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item2).Min()).Min();
-      var best = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item2).Max()).Max();
-      var maximization = IsMaximization();
-      if (!maximization) {
-        var h = worst;
-        worst = best;
-        best = h;
-      }
-
-      if (best == 0 || Math.Abs(best - worst) < Math.Abs(best * 2))
-        targets = Enumerable.Range(0, 11).Select(x => worst + (x / 10.0) * (best - worst)).ToArray();
-      else if (best > 0) targets = Enumerable.Range(0, 11).Select(x => best * (1.0 + (10 - x) / 10.0)).ToArray();
-      else if (best < 0) targets = Enumerable.Range(0, 11).Select(x => best / (1.0 + (10 - x) / 10.0)).ToArray();
-
+    private void GenerateDefaultTargets() {
+      targets = new[] { 0.1, 0.05, 0.02, 0.01, 0 };
       suppressTargetsEvents = true;
-      targetsTextBox.Text = string.Join(" ; ", targets);
+      targetsTextBox.Text = string.Join("% ; ", targets.Select(x => x * 100));
       suppressTargetsEvents = false;
     }
 
-    private void CalculateHitsForEachTarget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, bool maximization, int groupCount, string groupName) {
-      foreach (var l in targets) {
-        var key = groupName + "-" + l;
+    private void CalculateHitsForEachTarget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, bool maximization, string group, int groupCount, double bestTarget, int problemCount) {
+      foreach (var l in targets.Select(x => (maximization ? (1 - x) : (1 + x)) * bestTarget)) {
+        var key = group + "-" + l;
         if (!hits.ContainsKey(key)) hits.Add(key, new SortedList<double, double>());
         foreach (var v in row.Values) {
           if (maximization && v.Item2 >= l || !maximization && v.Item2 <= l) {
             if (hits[key].ContainsKey(v.Item1))
-              hits[key][v.Item1] += 1.0 / groupCount;
-            else hits[key][v.Item1] = 1.0 / groupCount;
+              hits[key][v.Item1] += 1.0 / (groupCount * problemCount);
+            else hits[key][v.Item1] = 1.0 / (groupCount * problemCount);
             break;
           }
@@ -405 +411 @@
     }
 
-    private double CalculateHitsForAllTargets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, bool maximization, int groupCount, string groupName) {
+    private double CalculateHitsForAllTargets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, bool maximization, string group, int groupCount, double bestTarget, int problemCount) {
       var values = row.Values;
-      if (!hits.ContainsKey(groupName)) hits.Add(groupName, new SortedList<double, double>());
+      if (!hits.ContainsKey(group)) hits.Add(group, new SortedList<double, double>());
 
       var i = 0;
       var j = 0;
       while (i < targets.Length && j < values.Count) {
+        var target = (maximization ? (1 - targets[i]) : (1 + targets[i])) * bestTarget;
         var current = values[j];
-        if (maximization && current.Item2 >= targets[i]
-          || !maximization && current.Item2 <= targets[i]) {
-          if (!hits[groupName].ContainsKey(current.Item1)) hits[groupName][current.Item1] = 0;
-          hits[groupName][current.Item1] += 1.0 / (groupCount * targets.Length);
+        if (maximization && current.Item2 >= target
+          || !maximization && current.Item2 <= target) {
+          if (!hits[group].ContainsKey(current.Item1)) hits[group][current.Item1] = 0;
+          hits[group][current.Item1] += 1.0 / (groupCount * problemCount * targets.Length);
           i++;
         } else {
@@ -426 +433 @@
     }
 
-    private void UpdateErtTables(List<Tuple<string, List<IRun>>> groupedRuns) {
+    private void UpdateErtTables(Dictionary<string, Dictionary<string, Tuple<double, List<IRun>>>> groupedRuns) {
       ertViewHost.Content = null;
       var columns = 1 + targets.Length + 1;
-      var matrix = new string[groupedRuns.Count + 1, columns];
+      var matrix = new string[groupedRuns.Count * groupedRuns.Max(x => x.Value.Count) + groupedRuns.Max(x => x.Value.Count), columns];
       var rowCount = 0;
       var maximization = IsMaximization();
+
+
+      var targetsPerProblem = (from r in Content
+                               let pd = new ProblemDescription(r).ToString()
+                               let target = r.Parameters.ContainsKey("BestKnownQuality")
+                                && r.Parameters["BestKnownQuality"] is DoubleValue
+                                  ? ((DoubleValue)r.Parameters["BestKnownQuality"]).Value
+                                  : ((IndexedDataTable<double>)r.Results[(string)dataTableComboBox.SelectedItem]).Rows.First().Values.Last().Item2
+                               group target by pd into g
+                               select new { Problem = g.Key, Target = maximization ? g.Max() : g.Min() })
+                              .ToDictionary(x => x.Problem, x => x.Target);
 
       var colNames = new string[columns];
       colNames[0] = colNames[colNames.Length - 1] = string.Empty;
       for (var i = 0; i < targets.Length; i++) {
-        colNames[i + 1] = maximization ? (targets[targets.Length - 1] / targets[i] - 1).ToString("0.0%")
-                                       : (targets[i] / targets[targets.Length - 1] - 1).ToString("0.0%");
-        matrix[rowCount, i + 1] = targets[i].ToString(CultureInfo.CurrentCulture.NumberFormat);
-      }
-      matrix[rowCount, columns - 1] = "#succ";
-      rowCount++;
-
-      foreach (var group in groupedRuns) {
-        matrix[rowCount, 0] = group.Item1;
-        var bestSucc = string.Empty;
+        colNames[i + 1] = targets[i].ToString("0.0%");
+      }
+      var problems = groupedRuns.SelectMany(x => x.Value.Keys).Distinct().ToList();
+
+      foreach (var problem in problems) {
+        matrix[rowCount, 0] = problem;
         for (var i = 0; i < targets.Length; i++) {
-          string succ;
-          matrix[rowCount, i + 1] = CalculateExpectedRunTime(group.Item2, targets[i], maximization, out succ);
-          if (i == targets.Length - 1) bestSucc = succ;
-        }
-        matrix[rowCount, columns - 1] = bestSucc;
+          matrix[rowCount, i + 1] = (targetsPerProblem[problem] * (maximization ? (1 - targets[i]) : (1 + targets[i]))).ToString(CultureInfo.CurrentCulture.NumberFormat);
+        }
+        matrix[rowCount, columns - 1] = "#succ";
         rowCount++;
+
+        foreach (var group in groupedRuns) {
+          matrix[rowCount, 0] = group.Key;
+          var runs = group.Value[problem].Item2;
+          var bestSucc = string.Empty;
+          for (var i = 0; i < targets.Length; i++) {
+            string succ;
+            matrix[rowCount, i + 1] = CalculateExpectedRunTime(runs, (maximization ? (1 - targets[i]) : (1 + targets[i])) * group.Value[problem].Item1, maximization, out succ);
+            if (i == targets.Length - 1) bestSucc = succ;
+          }
+          matrix[rowCount, columns - 1] = bestSucc;
+          rowCount++;
+        }
       }
       ertViewHost.Content = new StringMatrix(matrix) { ColumnNames = colNames };
@@ -500 +525 @@
       var colorCount = 0;
       var lineStyleCount = 0;
-      var maximization = IsMaximization();
 
       foreach (var group in groupedRuns) {
         var hits = new Dictionary<string, SortedList<double, double>>();
 
-        foreach (var run in group.Item2) {
-          var resultsTable = (IndexedDataTable<double>)run.Results[table];
-
-          if (eachOrAllBudgetsCheckBox.Checked) {
-            CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), maximization, group.Item2.Count, group.Item1);
-          } else {
-            CalculateHitsForAllBudgets(hits, resultsTable.Rows.First(), maximization, group.Item2.Count, group.Item1);
+        foreach (var problem in group.Value) {
+          foreach (var run in problem.Value.Item2) {
+            var resultsTable = (IndexedDataTable<double>)run.Results[table];
+
+            if (eachOrAllBudgetsCheckBox.Checked) {
+              CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), group.Value.Count, group.Key, problem.Value.Item2.Count);
+            } else {
+              CalculateHitsForAllBudgets(hits, resultsTable.Rows.First(), group.Value.Count, group.Key, problem.Value.Item2.Count);
+            }
           }
         }
@@ -542 +568 @@
 
     private void GenerateDefaultBudgets(string table) {
-      var runs = GroupRuns().SelectMany(x => x.Item2).ToList();
+      var runs = GroupRuns().SelectMany(x => x.Value.Values).SelectMany(x => x.Item2).ToList();
       var min = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Min()).Min();
       var max = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Max()).Max();
@@ -562 +588 @@
     }
 
-    private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, bool maximization, int groupCount, string groupName) {
+    private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, string groupName, int problemCount) {
       foreach (var b in budgets) {
         var key = groupName + "-" + b;
@@ -573 +599 @@
             var tgt = (prev == null || v.Item1 == b) ? v.Item2 : prev.Item2;
             if (hits[key].ContainsKey(tgt))
-              hits[key][tgt] += 1.0 / groupCount;
-            else hits[key][tgt] = 1.0 / groupCount;
+              hits[key][tgt] += 1.0 / (groupCount * problemCount);
+            else hits[key][tgt] = 1.0 / (groupCount * problemCount);
             break;
           }
@@ -583 +609 @@
     }
 
-    private void CalculateHitsForAllBudgets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, bool maximization, int groupCount, string groupName) {
+    private void CalculateHitsForAllBudgets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, int groupCount, string groupName, int problemCount) {
       var values = row.Values;
       if (!hits.ContainsKey(groupName)) hits.Add(groupName, new SortedList<double, double>());
@@ -596 +622 @@
             var tgt = (prev == null || current.Item1 == budgets[i]) ? current.Item2 : prev.Item2;
             if (!hits[groupName].ContainsKey(tgt)) hits[groupName][tgt] = 0;
-            hits[groupName][tgt] += 1.0 / (groupCount * budgets.Length);
+            hits[groupName][tgt] += 1.0 / (groupCount * problemCount * budgets.Length);
           }
           i++;
@@ -607 +633 @@
       if (i < budgets.Length && !hits[groupName].ContainsKey(lastTgt)) hits[groupName][lastTgt] = 0;
       while (i < budgets.Length) {
-        hits[groupName][lastTgt] += 1.0 / (groupCount * budgets.Length);
+        hits[groupName][lastTgt] += 1.0 / (groupCount * problemCount * budgets.Length);
         i++;
       }
@@ -622 +648 @@
     }
     private void problemComboBox_SelectedIndexChanged(object sender, EventArgs e) {
-      if (dataTableComboBox.SelectedIndex >= 0)
-        GenerateDefaultTargets((string)dataTableComboBox.SelectedItem);
       UpdateRuns();
       SetEnabledStateOfControls();
@@ -643 +667 @@
     private void targetsTextBox_Validating(object sender, CancelEventArgs e) {
       if (suppressTargetsEvents) return;
-      var targetStrings = targetsTextBox.Text.Split(new[] { ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
-      var targetList = new List<double>();
+      var targetStrings = targetsTextBox.Text.Split(new[] { '%', ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
+      var targetList = new List<decimal>();
       foreach (var ts in targetStrings) {
-        double t;
-        if (!double.TryParse(ts, out t)) {
+        decimal t;
+        if (!decimal.TryParse(ts, out t)) {
           errorProvider.SetError(targetsTextBox, "Not all targets can be parsed: " + ts);
           e.Cancel = true;
           return;
         }
-        targetList.Add(t);
+        targetList.Add(t / 100);
       }
       if (targetList.Count == 0) {
@@ -661 +685 @@
       e.Cancel = false;
       errorProvider.SetError(targetsTextBox, null);
-      targets = targetList.ToArray();
+      targets = targetList.Select(x => (double)x).ToArray();
       UpdateResultsByTarget();
       SetEnabledStateOfControls();
@@ -694 +718 @@
                                   : dialog.Values.Reverse().Select(x => (double)x).ToArray();
             suppressTargetsEvents = true;
-            targetsTextBox.Text = string.Join(" ; ", targets);
+            targetsTextBox.Text = string.Join("% ; ", targets);
             suppressTargetsEvents = false;
 
@@ -707 +731 @@
       var table = (string)dataTableComboBox.SelectedItem;
       var maximization = IsMaximization();
+
+      var targetsPerProblem = (from r in Content
+                               let pd = new ProblemDescription(r).ToString()
+                               let target = r.Parameters.ContainsKey("BestKnownQuality")
+                                && r.Parameters["BestKnownQuality"] is DoubleValue
+                                  ? ((DoubleValue)r.Parameters["BestKnownQuality"]).Value
+                                  : ((IndexedDataTable<double>)r.Results[table]).Rows.First().Values.Last().Item2
+                               group target by pd into g
+                               select new { Problem = g.Key, Target = maximization ? g.Max() : g.Min() })
+                              .ToDictionary(x => x.Problem, x => x.Target);
+
       foreach (var run in Content) {
         if (!run.Results.ContainsKey(table)) continue;
@@ -713 +748 @@
         var i = 0;
         var j = 0;
+        var pd = new ProblemDescription(run);
         while (i < targets.Length && j < values.Count) {
+          var target = (maximization ? (1 - targets[i]) : (1 + targets[i])) * targetsPerProblem[pd.ToString()];
           var current = values[j];
-          if (maximization && current.Item2 >= targets[i]
-              || !maximization && current.Item2 <= targets[i]) {
-            run.Results[table + ".Target" + targets[i]] = new DoubleValue(current.Item1);
+          if (maximization && current.Item2 >= target
+              || !maximization && current.Item2 <= target) {
+            run.Results[table + ".Target" + target] = new DoubleValue(current.Item1);
             i++;
           } else {
@@ -865 +902 @@
     private class ProblemDescription {
       private readonly bool matchAll;
-      public static readonly ProblemDescription MatchAll = new ProblemDescription();
+      public static readonly ProblemDescription MatchAll = new ProblemDescription() {
+        ProblemName = "All with Best-Known"
+      };
 
       private ProblemDescription() {