Changeset 13794 for branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.cs

Timestamp:

04/26/16 00:02:05 (8 years ago)

Author:

abeham

Message:

#2457: worked on recommendation algorithms (x-validation)

File:

• branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.cs (modified) (2 diffs)

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.cs

@@ -20 +20 @@
 #endregion
 
+using HeuristicLab.Analysis;
 using HeuristicLab.Common.Resources;
 using HeuristicLab.Core;
@@ -134 +135 @@
       var trainingSet = Content.ProblemInstances.Where(x => !Content.IsCurrentInstance(x)).ToArray();
 
+      var absErr = 0.0;
+      var confMatrix = new int[6, 6];
+      var tau = 0.0;
       // leave one out crossvalidation
       foreach (var pi in trainingSet) {
         var model = recommender.TrainModel(trainingSet.Where(x => x != pi).ToArray(), Content, features);
-        var ranking = model.GetRanking(pi).ToList();
-        var realPerformance = Content.GetAlgorithmPerformance(pi);
-        // absolute error predicting ert
-        // confusion matrix predicting class
-        // Kendall's tau
+        var predicted = model.GetRanking(pi).ToDictionary(x => x.Key, x => x.Value);
+        var observed = Content.GetAlgorithmPerformance(pi);
+        absErr += AbsoluteError(observed, predicted);
+        var confMat = ConfusionMatrix(observed, predicted);
+        for (var i = 0; i < confMat.GetLength(0); i++)
+          for (var j = 0; j < confMat.GetLength(1); j++)
+            confMatrix[i, j] += confMat[i, j];
+        tau += KendallsTau(observed, predicted);
         // average NCDG ... relevance determined by clustering (unsuccessful algorithms being penalized with negative relevance)
         // mean reciprocal rank
         // optional: expected reciprocal rank
       }
-    }
-
-    private static double AbsoluteError(Dictionary<IAlgorithm, double> performance, List<Tuple<IAlgorithm, double>> ranking) {
+      absErr /= trainingSet.Length;
+      tau /= trainingSet.Length;
+
+      confusionMatrixView.Content = new IntMatrix(confMatrix);
+      absoluteErrorView.Content = new DoubleValue(absErr);
+      kendallsTauView.Content = new DoubleValue(tau);
+    }
+
+    private static double AbsoluteError(Dictionary<IAlgorithm, double> performance, Dictionary<IAlgorithm, double> ranking) {
       var error = 0.0;
       foreach (var tuple in ranking) {
         double actual;
-        if (!performance.TryGetValue(tuple.Item1, out actual)) continue;
-        error += Math.Abs(actual - tuple.Item2);
+        if (!performance.TryGetValue(tuple.Key, out actual)) continue;
+        if (double.IsNaN(actual)) actual = int.MaxValue;
+        error += Math.Abs(actual - tuple.Value);
       }
       return error;
     }
 
-    private static int[,] ConfusionMatrix(Dictionary<IAlgorithm, double> performance, List<Tuple<IAlgorithm, double>> ranking) {
-      var confMatrix = new int[5,5];
+    private static int[,] ConfusionMatrix(Dictionary<IAlgorithm, double> performance, Dictionary<IAlgorithm, double> ranking) {
+      var confMatrix = new int[6,6];
+      var clusteredPerformance = ClusteringHelper<IAlgorithm>.Cluster(5, performance, a => double.IsNaN(a.Value)).GetByInstance().ToDictionary(x => x.Key, x => x.Value.Item2);
+      var clusteredRanks = ClusteringHelper<IAlgorithm>.Cluster(5, ranking, x => double.IsNaN(x.Value) || x.Value >= int.MaxValue).GetByInstance().ToDictionary(x => x.Key, x => x.Value.Item2);
+      foreach (var cr in clusteredRanks) {
+        int realRank;
+        if (!clusteredPerformance.TryGetValue(cr.Key, out realRank)) continue;
+        confMatrix[realRank, cr.Value]++;
+      }
       return confMatrix;
+    }
+
+    private static double KendallsTau(Dictionary<IAlgorithm, double> performance, Dictionary<IAlgorithm, double> ranking) {
+      var commonKeys = performance.Keys.Intersect(ranking.Keys).ToList();
+      var n = commonKeys.Count;
+      if (n == 0) return 0.0;
+
+      var actualRanks = commonKeys.Select((x, i) => new { Alg = x, Index = i, Perf = performance[x] })
+                                  .OrderBy(x => x.Perf).Select((x, i) => new { Index = x.Index, Rank = i })
+                                  .OrderBy(x => x.Index).Select(x => x.Rank).ToList();
+      var predictedRanks = commonKeys.Select((x, i) => new { Alg = x, Index = i, Perf = ranking[x] })
+                                     .OrderBy(x => x.Perf).Select((x, i) => new { Index = x.Index, Rank = i })
+                                     .OrderBy(x => x.Index).Select(x => x.Rank).ToList();
+      
+      var paired = actualRanks.Zip(predictedRanks, (a, p) => new { Actual = a, Predicted = p }).ToList();
+      var concordant = 0;
+      var discordant = 0;
+      for (var i = 0; i < paired.Count - 1; i++)
+        for (var j = 0; j < paired.Count; j++) {
+          if (paired[i].Actual > paired[j].Actual && paired[i].Predicted > paired[j].Predicted
+            || paired[i].Actual < paired[j].Actual && paired[i].Predicted < paired[j].Predicted)
+            concordant++;
+          else discordant++;
+        }
+      return (2.0 * (concordant - discordant)) / (n * (n - 1));
     }
     #endregion