Index: branches/PerformanceComparison/HeuristicLab.Analysis/3.3/QualityAnalysis/ExpectedRuntimeHelper.cs
===================================================================
--- branches/PerformanceComparison/HeuristicLab.Analysis/3.3/QualityAnalysis/ExpectedRuntimeHelper.cs	(revision 13799)
+++ branches/PerformanceComparison/HeuristicLab.Analysis/3.3/QualityAnalysis/ExpectedRuntimeHelper.cs	(revision 13803)
@@ -1,3 +1,4 @@
-﻿using HeuristicLab.Optimization;
+﻿using HeuristicLab.Common;
+using HeuristicLab.Optimization;
 using System;
 using System.Collections.Generic;
@@ -24,14 +25,17 @@
       }
 
-      var ert = double.NaN;
+      var ert = double.PositiveInfinity;
 
+      var nRuns = successful.Count + unsuccessful.Count;
       if (successful.Count > 0) {
-        if (unsuccessful.Count == 0) ert = successful.Average();
-        else {
-          var ps = successful.Count / (double)(successful.Count + unsuccessful.Count);
-          ert = successful.Average() + ((1.0 - ps) / ps) * unsuccessful.Average();
-        }
+        var succAvg = successful.Average();
+        var succDev = successful.StandardDeviation();
+        successful.RemoveAll(x => x < succAvg - 2 * succDev);
+        unsuccessful.RemoveAll(x => x < succAvg - 2 * succDev);
+        nRuns = successful.Count + unsuccessful.Count;
+
+        ert = successful.Average() / (successful.Count / (double)nRuns);
       }
-      return new ErtCalculationResult(successful.Count, (successful.Count + unsuccessful.Count), ert);
+      return new ErtCalculationResult(successful.Count, nRuns, ert);
     }
 
Index: branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/KnowledgeCenter.cs
===================================================================
--- branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/KnowledgeCenter.cs	(revision 13799)
+++ branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/KnowledgeCenter.cs	(revision 13803)
@@ -654,5 +654,5 @@
           var bkq = ((DoubleValue)prob.Parameters["BestKnownQuality"]).Value;
           var ert = ExpectedRuntimeHelper.CalculateErt(pr.ToList(), "QualityPerEvaluations", GetTarget(bkq, target, max), max).ExpectedRuntime;
-          if (double.IsNaN(ert)) ert = int.MaxValue;
+          if (double.IsInfinity(ert)) ert = int.MaxValue;
           ds.AddRow(new object[] { pr.Key }.Concat(f.Cast<object>()).Concat(new object[] { ert }));
         }
@@ -720,5 +720,5 @@
         var values = pr.GroupBy(x => algorithmId2RunMapping.GetBySecond(x).Single())
                        .ToDictionary(x => x.Key, x => ExpectedRuntimeHelper.CalculateErt(x.ToList(), "QualityPerEvaluations", GetTarget(bkq, target, max), max).ExpectedRuntime);
-        var ranks = ClusteringHelper<long>.Cluster(nClasses, values, x => double.IsNaN(x.Value))
+        var ranks = ClusteringHelper<long>.Cluster(nClasses, values, x => double.IsInfinity(x.Value))
           .GetByCluster().ToList();
         foreach (var c in ranks) {
Index: branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/KNearestNeighborModel.cs
===================================================================
--- branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/KNearestNeighborModel.cs	(revision 13799)
+++ branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/KNearestNeighborModel.cs	(revision 13803)
@@ -49,9 +49,11 @@
       var feature = KnowledgeCenter.GetFeatures(new [] { problemInstance }, characteristics, medianValues)[0];
       var nearestK = features.Select((f, i) => new { ProblemInstanceIndex = i, Feature = f })
-                             .OrderBy(x => x.Feature.Select((f, i) => (f - feature[i]) * (f - feature[i])).Sum())
-                             .Take(K);
+                             .OrderBy(x => x.Feature.Select((f, i) => (f - feature[i]) * (f - feature[i])).Sum());
       
-      var performances = new Dictionary<IAlgorithm, List<double>>();
+      var performances = new Dictionary<IAlgorithm, Performance>();
+
+      var k = 0;
       foreach (var next in nearestK) {
+        if (k >= K) break;
         var perfs = performance[problemInstanceMap.GetByFirst(next.ProblemInstanceIndex)];
         if (perfs.Count == 0) continue;
@@ -59,16 +61,39 @@
         foreach (var p in perfs) {
           var ert = p.Value;
-          if (double.IsNaN(ert)) ert = int.MaxValue;
-          List<double> erts;
-          if (!performances.TryGetValue(p.Key, out erts)) {
-            performances[p.Key] = new List<double>() { ert }; ;
-          } else erts.Add(ert);
+          Performance perf;
+          if (!performances.TryGetValue(p.Key, out perf)) {
+            perf = new Performance();
+            performances[p.Key] = perf;
+          }
+          perf.Add(ert);
         }
+
+        k++;
       }
 
-      return performances.Select(x => new { Alg = x.Key, Perf = x.Value.Average() })
+      return performances.Select(x => new { Alg = x.Key, Perf = x.Value.ExpectedRuntime() })
                          .OrderBy(x => x.Perf)
                          .Select(x => new KeyValuePair<IAlgorithm, double>(x.Alg, x.Perf));
     }
+
+    private class Performance {
+      private readonly List<double> successful;
+      private int runs;
+      public int Fails { get { return runs - successful.Count; } }
+
+      public Performance() {
+        successful = new List<double>();
+      }
+
+      public void Add(double ert) {
+        if (!double.IsInfinity(ert)) successful.Add(ert);
+        runs++;
+      }
+
+      public double ExpectedRuntime() {
+        if (successful.Count == 0) return int.MaxValue;
+        return successful.Average() / (successful.Count / (double)runs);
+      }
+    }
   }
 }
Index: branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/OverallBestRecommender.cs
===================================================================
--- branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/OverallBestRecommender.cs	(revision 13799)
+++ branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/OverallBestRecommender.cs	(revision 13803)
@@ -63,5 +63,5 @@
           if (!pis.TryGetValue(problemRuns.Key, out bkq)) continue;
           var ert = ExpectedRuntimeHelper.CalculateErt(problemRuns.ToList(), "QualityPerEvaluations", kc.GetTarget(bkq, kc.MinimumTarget.Value, kc.Maximization), kc.Maximization).ExpectedRuntime;
-          if (double.IsNaN(ert)) ert = int.MaxValue;
+          if (double.IsInfinity(ert)) ert = int.MaxValue;
           avgERT += ert;
           count++;
Index: branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.Designer.cs
===================================================================
--- branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.Designer.cs	(revision 13799)
+++ branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.Designer.cs	(revision 13803)
@@ -55,7 +55,8 @@
       this.parameterCollectionView = new HeuristicLab.Core.Views.ParameterCollectionView();
       this.crossvalidationTabPage = new System.Windows.Forms.TabPage();
+      this.topNComboBox = new System.Windows.Forms.ComboBox();
+      this.topNlabel = new System.Windows.Forms.Label();
       this.kendallsTauLabel = new System.Windows.Forms.Label();
       this.predictedLabel = new System.Windows.Forms.Label();
-      this.actualLabel = new System.Windows.Forms.Label();
       this.absoluteLogErrorLabel = new System.Windows.Forms.Label();
       this.absoluteErrorLabel = new System.Windows.Forms.Label();
@@ -67,4 +68,6 @@
       this.minTargetView = new HeuristicLab.Data.Views.StringConvertibleValueView();
       this.minimumTargetLabel = new System.Windows.Forms.Label();
+      this.ncdgView = new HeuristicLab.Data.Views.StringConvertibleValueView();
+      this.ndcgLabel = new System.Windows.Forms.Label();
       this.tabControl.SuspendLayout();
       this.characteristicsTabPage.SuspendLayout();
@@ -139,5 +142,5 @@
       this.characteristicsViewHost.Name = "characteristicsViewHost";
       this.characteristicsViewHost.ReadOnly = false;
-      this.characteristicsViewHost.Size = new System.Drawing.Size(680, 283);
+      this.characteristicsViewHost.Size = new System.Drawing.Size(940, 442);
       this.characteristicsViewHost.TabIndex = 1;
       this.characteristicsViewHost.ViewsLabelVisible = true;
@@ -180,9 +183,12 @@
       // crossvalidationTabPage
       // 
+      this.crossvalidationTabPage.Controls.Add(this.topNComboBox);
+      this.crossvalidationTabPage.Controls.Add(this.topNlabel);
+      this.crossvalidationTabPage.Controls.Add(this.ndcgLabel);
       this.crossvalidationTabPage.Controls.Add(this.kendallsTauLabel);
       this.crossvalidationTabPage.Controls.Add(this.predictedLabel);
-      this.crossvalidationTabPage.Controls.Add(this.actualLabel);
       this.crossvalidationTabPage.Controls.Add(this.absoluteLogErrorLabel);
       this.crossvalidationTabPage.Controls.Add(this.absoluteErrorLabel);
+      this.crossvalidationTabPage.Controls.Add(this.ncdgView);
       this.crossvalidationTabPage.Controls.Add(this.kendallsTauView);
       this.crossvalidationTabPage.Controls.Add(this.absoluteLogErrorView);
@@ -198,8 +204,27 @@
       this.crossvalidationTabPage.UseVisualStyleBackColor = true;
       // 
+      // topNComboBox
+      // 
+      this.topNComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
+      this.topNComboBox.FormattingEnabled = true;
+      this.topNComboBox.Location = new System.Drawing.Point(331, 9);
+      this.topNComboBox.Name = "topNComboBox";
+      this.topNComboBox.Size = new System.Drawing.Size(76, 21);
+      this.topNComboBox.TabIndex = 5;
+      this.topNComboBox.SelectedIndexChanged += new System.EventHandler(this.topNComboBox_SelectedIndexChanged);
+      // 
+      // topNlabel
+      // 
+      this.topNlabel.AutoSize = true;
+      this.topNlabel.Location = new System.Drawing.Point(192, 12);
+      this.topNlabel.Name = "topNlabel";
+      this.topNlabel.Size = new System.Drawing.Size(133, 13);
+      this.topNlabel.TabIndex = 4;
+      this.topNlabel.Text = "Evaluate only best ranked:";
+      // 
       // kendallsTauLabel
       // 
       this.kendallsTauLabel.AutoSize = true;
-      this.kendallsTauLabel.Location = new System.Drawing.Point(519, 165);
+      this.kendallsTauLabel.Location = new System.Drawing.Point(519, 224);
       this.kendallsTauLabel.Margin = new System.Windows.Forms.Padding(3);
       this.kendallsTauLabel.Name = "kendallsTauLabel";
@@ -219,15 +244,4 @@
       this.predictedLabel.Text = "PREDICTED";
       // 
-      // actualLabel
-      // 
-      this.actualLabel.AutoSize = true;
-      this.actualLabel.Font = new System.Drawing.Font("Microsoft Sans Serif", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
-      this.actualLabel.Location = new System.Drawing.Point(16, 155);
-      this.actualLabel.Margin = new System.Windows.Forms.Padding(3);
-      this.actualLabel.Name = "actualLabel";
-      this.actualLabel.Size = new System.Drawing.Size(15, 78);
-      this.actualLabel.TabIndex = 3;
-      this.actualLabel.Text = "A\r\nC\r\nT\r\nU\r\nA\r\nL";
-      // 
       // absoluteLogErrorLabel
       // 
@@ -236,7 +250,7 @@
       this.absoluteLogErrorLabel.Margin = new System.Windows.Forms.Padding(3);
       this.absoluteLogErrorLabel.Name = "absoluteLogErrorLabel";
-      this.absoluteLogErrorLabel.Size = new System.Drawing.Size(127, 13);
+      this.absoluteLogErrorLabel.Size = new System.Drawing.Size(162, 13);
       this.absoluteLogErrorLabel.TabIndex = 3;
-      this.absoluteLogErrorLabel.Text = "Mean Absolute Log Error:";
+      this.absoluteLogErrorLabel.Text = "Mean Absolute Log Error (top N):";
       // 
       // absoluteErrorLabel
@@ -246,7 +260,7 @@
       this.absoluteErrorLabel.Margin = new System.Windows.Forms.Padding(3);
       this.absoluteErrorLabel.Name = "absoluteErrorLabel";
-      this.absoluteErrorLabel.Size = new System.Drawing.Size(106, 13);
+      this.absoluteErrorLabel.Size = new System.Drawing.Size(141, 13);
       this.absoluteErrorLabel.TabIndex = 3;
-      this.absoluteErrorLabel.Text = "Mean Absolute Error:";
+      this.absoluteErrorLabel.Text = "Mean Absolute Error (top N):";
       // 
       // kendallsTauView
@@ -255,5 +269,5 @@
       this.kendallsTauView.Content = null;
       this.kendallsTauView.LabelVisible = false;
-      this.kendallsTauView.Location = new System.Drawing.Point(503, 184);
+      this.kendallsTauView.Location = new System.Drawing.Point(503, 243);
       this.kendallsTauView.Name = "kendallsTauView";
       this.kendallsTauView.ReadOnly = true;
@@ -292,5 +306,5 @@
       this.confusionMatrixView.ShowRowsAndColumnsTextBox = false;
       this.confusionMatrixView.ShowStatisticalInformation = false;
-      this.confusionMatrixView.Size = new System.Drawing.Size(438, 244);
+      this.confusionMatrixView.Size = new System.Drawing.Size(438, 134);
       this.confusionMatrixView.TabIndex = 1;
       // 
@@ -324,4 +338,25 @@
       this.minimumTargetLabel.TabIndex = 17;
       this.minimumTargetLabel.Text = "Target:";
+      // 
+      // ncdgView
+      // 
+      this.ncdgView.Caption = "StringConvertibleValue View";
+      this.ncdgView.Content = null;
+      this.ncdgView.LabelVisible = false;
+      this.ncdgView.Location = new System.Drawing.Point(503, 184);
+      this.ncdgView.Name = "ncdgView";
+      this.ncdgView.ReadOnly = true;
+      this.ncdgView.Size = new System.Drawing.Size(194, 21);
+      this.ncdgView.TabIndex = 2;
+      // 
+      // ndcgLabel
+      // 
+      this.ndcgLabel.AutoSize = true;
+      this.ndcgLabel.Location = new System.Drawing.Point(519, 165);
+      this.ndcgLabel.Margin = new System.Windows.Forms.Padding(3);
+      this.ndcgLabel.Name = "ndcgLabel";
+      this.ndcgLabel.Size = new System.Drawing.Size(106, 13);
+      this.ndcgLabel.TabIndex = 3;
+      this.ndcgLabel.Text = "Mean NDCG (top N):";
       // 
       // PerformanceModelingView
@@ -368,7 +403,10 @@
     private Data.Views.StringConvertibleMatrixView confusionMatrixView;
     private System.Windows.Forms.Label predictedLabel;
-    private System.Windows.Forms.Label actualLabel;
     private System.Windows.Forms.Label absoluteLogErrorLabel;
     private Data.Views.StringConvertibleValueView absoluteLogErrorView;
+    private System.Windows.Forms.ComboBox topNComboBox;
+    private System.Windows.Forms.Label topNlabel;
+    private System.Windows.Forms.Label ndcgLabel;
+    private Data.Views.StringConvertibleValueView ncdgView;
   }
 }
Index: branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.cs
===================================================================
--- branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.cs	(revision 13799)
+++ branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.cs	(revision 13803)
@@ -59,4 +59,5 @@
       }
       UpdateCharacteristics();
+      UpdateTopNCombobox();
     }
 
@@ -66,5 +67,5 @@
       recommendStartButton.Enabled = Content != null && recommenderComboBox.SelectedIndex >= 0 && characteristics.CheckedItems.Any();
       characteristicsViewHost.Enabled = Content != null;
-      xValidateButton.Enabled = Content != null && recommenderComboBox.SelectedIndex >= 0 && characteristics.CheckedItems.Any();
+      xValidateButton.Enabled = Content != null && recommenderComboBox.SelectedIndex >= 0 && characteristics.CheckedItems.Any() && topNComboBox.SelectedIndex >= 0;
     }
 
@@ -101,4 +102,19 @@
       }
     }
+
+    private void UpdateTopNCombobox() {
+      if (InvokeRequired) { Invoke((Action)UpdateTopNCombobox); return; }
+
+      int selected = 3;
+      if (topNComboBox.SelectedIndex >= 0) selected = (int)topNComboBox.SelectedItem;
+      topNComboBox.Items.Clear();
+      if (Content == null) return;
+
+      var algInstances = Content.AlgorithmInstances.Count;
+      for (var i = 1; i <= algInstances; i++) {
+        topNComboBox.Items.Add(i);
+      }
+      topNComboBox.SelectedIndex = Math.Min(selected, topNComboBox.Items.Count) - 1;
+    }
     #endregion
 
@@ -108,4 +124,9 @@
       UpdateCharacteristics();
       SetEnabledStateOfControls();
+    }
+
+    protected override void OnAlgorithmInstancesChanged() {
+      base.OnAlgorithmInstancesChanged();
+      UpdateTopNCombobox();
     }
     #endregion
@@ -133,79 +154,118 @@
       var recommender = (IAlgorithmInstanceRecommender)recommenderComboBox.SelectedItem;
       var progress = MainForm.AddOperationProgressToView(this, "Performing Leave-one-out Crossvalidation");
-
-      Task.Factory.StartNew(() => { DoCrossvalidate(recommender, progress); }, TaskCreationOptions.LongRunning);
-    }
-
-    private void DoCrossvalidate(IAlgorithmInstanceRecommender recommender, IProgress progress) {
-      var features = characteristics.CheckedItems.Select(x => x.Value.Value).ToArray();
-      var trainingSet = Content.ProblemInstances.Where(x => !Content.IsCurrentInstance(x)).ToArray();
-
-      var absErr = 0.0;
-      var absLogError = 0.0;
-      var confMatrix = new int[6, 6];
-      var tau = 0.0;
-      // leave one out crossvalidation
-      var count = 0;
-      foreach (var pi in trainingSet) {
-        progress.Status = pi.Name + "...";
-        var model = recommender.TrainModel(trainingSet.Where(x => x != pi).ToArray(), Content, features);
-        var predicted = model.GetRanking(pi).ToDictionary(x => x.Key, x => x.Value);
-        var observed = Content.GetAlgorithmPerformance(pi);
-        absErr += AbsoluteError(observed, predicted);
-        absLogError += AbsoluteLogError(observed, predicted);
-        var confMat = ConfusionMatrix(observed, predicted);
-        for (var i = 0; i < confMat.GetLength(0); i++) {
-          for (var j = 0; j < confMat.GetLength(1); j++)
-            confMatrix[i, j] += confMat[i, j];
+      var topN = (int)topNComboBox.SelectedItem;
+      Task.Factory.StartNew(() => { DoCrossvalidate(recommender, topN, progress); }, TaskCreationOptions.LongRunning);
+    }
+
+    private void topNComboBox_SelectedIndexChanged(object sender, EventArgs e) {
+      SetEnabledStateOfControls();
+    }
+    #endregion
+
+    #region Other Event Handlers
+    private void CharacteristicsOnCheckedItemsChanged(object sender, EventArgs e) {
+      SetEnabledStateOfControls();
+    }
+    #endregion
+    
+    private void DoCrossvalidate(IAlgorithmInstanceRecommender recommender, int topN, IProgress progress) {
+      try {
+        var features = characteristics.CheckedItems.Select(x => x.Value.Value).ToArray();
+        var trainingSet = Content.ProblemInstances.Where(x => !Content.IsCurrentInstance(x)).ToArray();
+
+        var absErr = 0.0;
+        var absErrCnt = 0;
+        var absLogErr = 0.0;
+        var absLogErrCnt = 0;
+        var confMatrix = new int[1, 6];
+        var tau = 0.0;
+        var tauCnt = 0;
+        var ndcg = 0.0;
+        var ndcgCnt = 0;
+        // leave one out crossvalidation
+        var count = 0;
+        foreach (var pi in trainingSet) {
+          var observed = Content.GetAlgorithmPerformance(pi);
+          if (observed.Count == 0) continue;
+          progress.Status = pi.Name + "...";
+          var model = recommender.TrainModel(trainingSet.Where(x => x != pi).ToArray(), Content, features);
+          var predictedTopN = model.GetRanking(pi).Take(topN).ToDictionary(x => x.Key, x => x.Value);
+          var predicted = model.GetRanking(pi).ToDictionary(x => x.Key, x => x.Value);
+          var ae = AbsoluteError(observed, predictedTopN);
+          if (!double.IsNaN(ae)) {
+            absErr += ae; // in case we only predicted instances that have not been observed
+            absErrCnt++;
+          }
+          var ale = AbsoluteLogError(observed, predictedTopN);
+          if (!double.IsNaN(ale)) {
+            absLogErr += ale; // in case we only predicted instances that have not been observed
+            absLogErrCnt++;
+          }
+          var confMat = ConfusionMatrix(observed, predictedTopN);
+          for (var i = 0; i < confMat.Length; i++) {
+            confMatrix[0, i] += confMat[i];
+          }
+          var kt = KendallsTau(observed, predicted);
+          if (!double.IsNaN(kt)) {
+            tau += kt;
+            tauCnt++;
+          }
+          var gain = NDCG(observed, model.GetRanking(pi).Take(topN).Select(x => x.Key).ToList());
+          if (!double.IsNaN(gain)) {
+            ndcg += gain;
+            ndcgCnt++;
+          }
+          // mean reciprocal rank
+          // optional: expected reciprocal rank
+          progress.ProgressValue = ++count / (double)trainingSet.Length;
         }
-        tau += KendallsTau(observed, predicted);
-        // average NCDG ... relevance determined by clustering (unsuccessful algorithms being penalized with negative relevance)
-        // mean reciprocal rank
-        // optional: expected reciprocal rank
-        progress.ProgressValue = ++count / (double)trainingSet.Length;
-      }
-      absErr /= trainingSet.Length;
-      absLogError /= trainingSet.Length;
-      tau /= trainingSet.Length;
-
-      absoluteErrorView.Content = new DoubleValue(absErr);
-      absoluteLogErrorView.Content = new DoubleValue(absLogError);
-      var description = new[] { "A", "B", "C", "D", "E", "F" };
-      confusionMatrixView.Content = new IntMatrix(confMatrix) { ColumnNames = description, RowNames = description };
-      kendallsTauView.Content = new DoubleValue(tau);
-
-      progress.Finish();
+        absErr /= absErrCnt;
+        absLogErr /= absLogErrCnt;
+        tau /= tauCnt;
+        ndcg /= ndcgCnt;
+
+        absoluteErrorView.Content = new DoubleValue(absErr);
+        absoluteLogErrorView.Content = new DoubleValue(absLogErr);
+        var description = new[] {"A", "B", "C", "D", "E", "F"};
+        confusionMatrixView.Content = new IntMatrix(confMatrix) {ColumnNames = description};
+        kendallsTauView.Content = new DoubleValue(tau);
+        ncdgView.Content = new DoubleValue(ndcg);
+      } finally { progress.Finish(); }
     }
 
     private static double AbsoluteError(Dictionary<IAlgorithm, double> performance, Dictionary<IAlgorithm, double> ranking) {
       var error = 0.0;
+      var count = 0;
       foreach (var tuple in ranking) {
         double actual;
         if (!performance.TryGetValue(tuple.Key, out actual)) continue;
-        if (double.IsNaN(actual)) actual = int.MaxValue;
+        if (double.IsInfinity(actual)) actual = int.MaxValue;
         error += Math.Abs(actual - tuple.Value);
-      }
-      return error;
+        count++;
+      }
+      return error / count;
     }
 
     private static double AbsoluteLogError(Dictionary<IAlgorithm, double> performance, Dictionary<IAlgorithm, double> ranking) {
       var error = 0.0;
+      var count = 0;
       foreach (var tuple in ranking) {
         double actual;
         if (!performance.TryGetValue(tuple.Key, out actual)) continue;
-        if (double.IsNaN(actual)) actual = int.MaxValue;
+        if (double.IsInfinity(actual)) actual = int.MaxValue;
         error += Math.Abs(Math.Log10(actual) - Math.Log10(tuple.Value));
-      }
-      return error;
-    }
-
-    private static int[,] ConfusionMatrix(Dictionary<IAlgorithm, double> performance, Dictionary<IAlgorithm, double> ranking) {
-      var confMatrix = new int[6,6];
-      var clusteredPerformance = ClusteringHelper<IAlgorithm>.Cluster(5, performance, a => double.IsNaN(a.Value)).GetByInstance().ToDictionary(x => x.Key, x => x.Value.Item2);
-      var clusteredRanks = ClusteringHelper<IAlgorithm>.Cluster(5, ranking, x => double.IsNaN(x.Value) || x.Value >= int.MaxValue).GetByInstance().ToDictionary(x => x.Key, x => x.Value.Item2);
-      foreach (var cr in clusteredRanks) {
+        count++;
+      }
+      return error / count;
+    }
+
+    private static int[] ConfusionMatrix(Dictionary<IAlgorithm, double> performance, Dictionary<IAlgorithm, double> ranking) {
+      var confMatrix = new int[6];
+      var clusteredPerformance = ClusteringHelper<IAlgorithm>.Cluster(5, performance, a => double.IsInfinity(a.Value)).GetByInstance().ToDictionary(x => x.Key, x => x.Value.Item2);
+      
+      foreach (var cr in ranking) {
         int realRank;
         if (!clusteredPerformance.TryGetValue(cr.Key, out realRank)) continue;
-        confMatrix[realRank, cr.Value]++;
+        confMatrix[realRank]++;
       }
       return confMatrix;
@@ -215,5 +275,5 @@
       var commonKeys = performance.Keys.Intersect(ranking.Keys).ToList();
       var n = commonKeys.Count;
-      if (n == 0) return 0.0;
+      if (n == 0) return double.NaN;
 
       var actualRanks = commonKeys.Select((x, i) => new { Alg = x, Index = i, Perf = performance[x] })
@@ -223,10 +283,10 @@
                                      .OrderBy(x => x.Perf).Select((x, i) => new { Index = x.Index, Rank = i })
                                      .OrderBy(x => x.Index).Select(x => x.Rank).ToList();
-      
+
       var paired = actualRanks.Zip(predictedRanks, (a, p) => new { Actual = a, Predicted = p }).ToList();
       var concordant = 0;
       var discordant = 0;
       for (var i = 0; i < paired.Count - 1; i++)
-        for (var j = 0; j < paired.Count; j++) {
+        for (var j = i + 1; j < paired.Count; j++) {
           if (paired[i].Actual > paired[j].Actual && paired[i].Predicted > paired[j].Predicted
             || paired[i].Actual < paired[j].Actual && paired[i].Predicted < paired[j].Predicted)
@@ -236,11 +296,32 @@
       return (2.0 * (concordant - discordant)) / (n * (n - 1));
     }
-    #endregion
-
-    #region Other Event Handlers
-    private void CharacteristicsOnCheckedItemsChanged(object sender, EventArgs e) {
-      SetEnabledStateOfControls();
-    }
-    #endregion
+
+    private static double NDCG(Dictionary<IAlgorithm, double> performance, List<IAlgorithm> ranking) {
+      var k = 5;
+      var relevance = ClusteringHelper<IAlgorithm>.Cluster(k, performance, a => double.IsInfinity(a.Value)).GetByInstance().ToDictionary(x => x.Key, x => k - x.Value.Item2);
+      
+      var i = 0;
+      var dcgp = 0.0;
+      for (; i < ranking.Count; i++) {
+        int rel;
+        if (!relevance.TryGetValue(ranking[i], out rel))
+          continue;
+        dcgp = rel;
+        i++;
+        break;
+      }
+      for (; i < ranking.Count; i++) {
+        int rel;
+        if (!relevance.TryGetValue(ranking[i], out rel))
+          continue;
+        dcgp += rel / Math.Log(i + 1, 2);
+      }
+      var ideal = relevance.Select(x => x.Value).OrderByDescending(x => x).Take(ranking.Count).ToArray();
+      double idcgp = ideal[0];
+      for (i = 1; i < ideal.Length; i++)
+        idcgp += ideal[i] / Math.Log(i + 1, 2);
+
+      return dcgp / idcgp;
+    }
   }
 }
