Changeset 15736 for branches/1614_GeneralizedQAP/HeuristicLab.OptimizationExpertSystem/3.3/Views

Timestamp:

02/07/18 20:55:57 (7 years ago)

Author:

abeham

Message:

#1614: updates from last night ...

Location:

branches/1614_GeneralizedQAP/HeuristicLab.OptimizationExpertSystem/3.3/Views

Files:

• PerformanceModelingView.Designer.cs (modified) (8 diffs)
• PerformanceModelingView.cs (modified) (10 diffs)

branches/1614_GeneralizedQAP/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.Designer.cs

@@ -57 +57 @@
       this.topNComboBox = new System.Windows.Forms.ComboBox();
       this.topNlabel = new System.Windows.Forms.Label();
+      this.ndcgLabel = new System.Windows.Forms.Label();
       this.kendallsTauLabel = new System.Windows.Forms.Label();
       this.predictedLabel = new System.Windows.Forms.Label();
       this.absoluteLogErrorLabel = new System.Windows.Forms.Label();
       this.absoluteErrorLabel = new System.Windows.Forms.Label();
+      this.ncdgView = new HeuristicLab.Data.Views.StringConvertibleValueView();
       this.kendallsTauView = new HeuristicLab.Data.Views.StringConvertibleValueView();
       this.absoluteLogErrorView = new HeuristicLab.Data.Views.StringConvertibleValueView();
@@ -68 +70 @@
       this.minTargetView = new HeuristicLab.Data.Views.StringConvertibleValueView();
       this.minimumTargetLabel = new System.Windows.Forms.Label();
-      this.ncdgView = new HeuristicLab.Data.Views.StringConvertibleValueView();
-      this.ndcgLabel = new System.Windows.Forms.Label();
+      this.spearmansRhoView = new HeuristicLab.Data.Views.StringConvertibleValueView();
+      this.spearmansRhoLabel = new System.Windows.Forms.Label();
       this.tabControl.SuspendLayout();
       this.characteristicsTabPage.SuspendLayout();
@@ -186 +188 @@
       this.crossvalidationTabPage.Controls.Add(this.topNlabel);
       this.crossvalidationTabPage.Controls.Add(this.ndcgLabel);
+      this.crossvalidationTabPage.Controls.Add(this.spearmansRhoLabel);
       this.crossvalidationTabPage.Controls.Add(this.kendallsTauLabel);
       this.crossvalidationTabPage.Controls.Add(this.predictedLabel);
@@ -191 +194 @@
       this.crossvalidationTabPage.Controls.Add(this.absoluteErrorLabel);
       this.crossvalidationTabPage.Controls.Add(this.ncdgView);
+      this.crossvalidationTabPage.Controls.Add(this.spearmansRhoView);
       this.crossvalidationTabPage.Controls.Add(this.kendallsTauView);
       this.crossvalidationTabPage.Controls.Add(this.absoluteLogErrorView);
@@ -223 +227 @@
       this.topNlabel.Text = "Evaluate only best ranked:";
       // 
+      // ndcgLabel
+      // 
+      this.ndcgLabel.AutoSize = true;
+      this.ndcgLabel.Location = new System.Drawing.Point(519, 165);
+      this.ndcgLabel.Margin = new System.Windows.Forms.Padding(3);
+      this.ndcgLabel.Name = "ndcgLabel";
+      this.ndcgLabel.Size = new System.Drawing.Size(106, 13);
+      this.ndcgLabel.TabIndex = 3;
+      this.ndcgLabel.Text = "Mean NDCG (top N):";
+      // 
       // kendallsTauLabel
       // 
@@ -263 +277 @@
       this.absoluteErrorLabel.TabIndex = 3;
       this.absoluteErrorLabel.Text = "Mean Absolute Error (top N):";
+      // 
+      // ncdgView
+      // 
+      this.ncdgView.Caption = "StringConvertibleValue View";
+      this.ncdgView.Content = null;
+      this.ncdgView.LabelVisible = false;
+      this.ncdgView.Location = new System.Drawing.Point(503, 184);
+      this.ncdgView.Name = "ncdgView";
+      this.ncdgView.ReadOnly = true;
+      this.ncdgView.Size = new System.Drawing.Size(194, 21);
+      this.ncdgView.TabIndex = 2;
       // 
       // kendallsTauView
@@ -339 +364 @@
       this.minimumTargetLabel.Text = "Target:";
       // 
-      // ncdgView
-      // 
-      this.ncdgView.Caption = "StringConvertibleValue View";
-      this.ncdgView.Content = null;
-      this.ncdgView.LabelVisible = false;
-      this.ncdgView.Location = new System.Drawing.Point(503, 184);
-      this.ncdgView.Name = "ncdgView";
-      this.ncdgView.ReadOnly = true;
-      this.ncdgView.Size = new System.Drawing.Size(194, 21);
-      this.ncdgView.TabIndex = 2;
-      // 
-      // ndcgLabel
-      // 
-      this.ndcgLabel.AutoSize = true;
-      this.ndcgLabel.Location = new System.Drawing.Point(519, 165);
-      this.ndcgLabel.Margin = new System.Windows.Forms.Padding(3);
-      this.ndcgLabel.Name = "ndcgLabel";
-      this.ndcgLabel.Size = new System.Drawing.Size(106, 13);
-      this.ndcgLabel.TabIndex = 3;
-      this.ndcgLabel.Text = "Mean NDCG (top N):";
+      // spearmansRhoView
+      // 
+      this.spearmansRhoView.Caption = "StringConvertibleValue View";
+      this.spearmansRhoView.Content = null;
+      this.spearmansRhoView.LabelVisible = false;
+      this.spearmansRhoView.Location = new System.Drawing.Point(503, 303);
+      this.spearmansRhoView.Name = "spearmansRhoView";
+      this.spearmansRhoView.ReadOnly = true;
+      this.spearmansRhoView.Size = new System.Drawing.Size(194, 21);
+      this.spearmansRhoView.TabIndex = 2;
+      // 
+      // spearmansRhoLabel
+      // 
+      this.spearmansRhoLabel.AutoSize = true;
+      this.spearmansRhoLabel.Location = new System.Drawing.Point(519, 284);
+      this.spearmansRhoLabel.Margin = new System.Windows.Forms.Padding(3);
+      this.spearmansRhoLabel.Name = "spearmansRhoLabel";
+      this.spearmansRhoLabel.Size = new System.Drawing.Size(118, 13);
+      this.spearmansRhoLabel.TabIndex = 3;
+      this.spearmansRhoLabel.Text = "Mean Spearman\'s Rho:";
       // 
       // PerformanceModelingView
@@ -409 +434 @@
     private System.Windows.Forms.Label ndcgLabel;
     private Data.Views.StringConvertibleValueView ncdgView;
+    private System.Windows.Forms.Label spearmansRhoLabel;
+    private Data.Views.StringConvertibleValueView spearmansRhoView;
   }
 }

branches/1614_GeneralizedQAP/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.cs

@@ -20 +20 @@
 #endregion
 
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
 using HeuristicLab.Analysis;
 using HeuristicLab.Common.Resources;
@@ -28 +32 @@
 using HeuristicLab.OptimizationExpertSystem.Common;
 using HeuristicLab.PluginInfrastructure;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
 
 namespace HeuristicLab.OptimizationExpertSystem {
@@ -181 +181 @@
         var tau = 0.0;
         var tauCnt = 0;
+        var rho = 0.0;
+        var rhoCnt = 0;
         var ndcg = 0.0;
         var ndcgCnt = 0;
@@ -207 +209 @@
           }
           var kt = KendallsTau(observed, predicted);
-          if (!double.IsNaN(kt)) {
+          if (!double.IsNaN(kt) && !double.IsInfinity(kt)) {
             tau += kt;
             tauCnt++;
+          }
+          var sr = SpearmansRho(observed, predicted);
+          if (!double.IsNaN(sr)) {
+            rho += sr;
+            rhoCnt++;
           }
           var gain = NDCG(observed, model.GetRanking(pi).Take(topN).Select(x => x.Key).ToList());
@@ -216 +223 @@
             ndcgCnt++;
           }
-          // mean reciprocal rank
-          // optional: expected reciprocal rank
           progress.ProgressValue = ++count / (double)trainingSet.Length;
         }
@@ -223 +228 @@
         absLogErr /= absLogErrCnt;
         tau /= tauCnt;
+        rho /= rhoCnt;
         ndcg /= ndcgCnt;
 
@@ -230 +236 @@
         confusionMatrixView.Content = new IntMatrix(confMatrix) {ColumnNames = description};
         kendallsTauView.Content = new DoubleValue(tau);
+        spearmansRhoView.Content = new DoubleValue(rho);
         ncdgView.Content = new DoubleValue(ndcg);
       } finally { progress.Finish(); }
@@ -272 +279 @@
     }
 
+    private static double SpearmansRho(Dictionary<IAlgorithm, double> performance, Dictionary<IAlgorithm, double> ranking) {
+      var commonKeys = performance.Keys.Intersect(ranking.Keys).ToList();
+      var n = commonKeys.Count;
+      if (n == 0) return double.NaN;
+
+      var actualRanks = commonKeys.Select((x, i) => new { Alg = x, Index = i, Perf = performance[x] })
+                                  .OrderBy(x => x.Perf).Select((x, i) => new { Index = x.Index, Rank = double.IsPositiveInfinity(x.Perf) ? int.MaxValue : i })
+                                  .OrderBy(x => x.Index).Select(x => (double)x.Rank).ToArray();
+      var predictedRanks = commonKeys.Select((x, i) => new { Alg = x, Index = i, Perf = ranking[x] })
+                                     .OrderBy(x => x.Perf).Select((x, i) => new { Index = x.Index, Rank = double.IsPositiveInfinity(x.Perf) ? int.MaxValue : i })
+                                     .OrderBy(x => x.Index).Select(x => (double)x.Rank).ToArray();
+
+      return alglib.spearmancorr2(actualRanks, predictedRanks);
+    }
+
     private static double KendallsTau(Dictionary<IAlgorithm, double> performance, Dictionary<IAlgorithm, double> ranking) {
       var commonKeys = performance.Keys.Intersect(ranking.Keys).ToList();
@@ -278 +300 @@
 
       var actualRanks = commonKeys.Select((x, i) => new { Alg = x, Index = i, Perf = performance[x] })
-                                  .OrderBy(x => x.Perf).Select((x, i) => new { Index = x.Index, Rank = i })
+                                  .OrderBy(x => x.Perf).Select((x, i) => new { Index = x.Index, Rank = double.IsPositiveInfinity(x.Perf) ? int.MaxValue : i })
                                   .OrderBy(x => x.Index).Select(x => x.Rank).ToList();
       var predictedRanks = commonKeys.Select((x, i) => new { Alg = x, Index = i, Perf = ranking[x] })
-                                     .OrderBy(x => x.Perf).Select((x, i) => new { Index = x.Index, Rank = i })
+                                     .OrderBy(x => x.Perf).Select((x, i) => new { Index = x.Index, Rank = double.IsPositiveInfinity(x.Perf) ? int.MaxValue : i })
                                      .OrderBy(x => x.Index).Select(x => x.Rank).ToList();
 
@@ -292 +314 @@
             || paired[i].Actual < paired[j].Actual && paired[i].Predicted < paired[j].Predicted)
             concordant++;
-          else discordant++;
+          else if (paired[i].Actual > paired[j].Actual && paired[i].Predicted < paired[j].Predicted
+            || paired[i].Actual < paired[j].Actual && paired[i].Predicted > paired[j].Predicted)
+            discordant++;
         }
-      return (2.0 * (concordant - discordant)) / (n * (n - 1));
+
+      var ti = performance.GroupBy(x => x.Value).Sum(x => x.Count() * (x.Count() - 1));
+      var uj = ranking.GroupBy(x => x.Value).Sum(x => x.Count() * (x.Count() - 1));
+      return (2.0 * (concordant - discordant)) / Math.Sqrt( (n * (n - 1) - ti * (ti - 1)) * (n * (n - 1) - uj * (uj - 1)) );
     }