Index: branches/PerformanceComparison/HeuristicLab.Analysis/3.3/Clustering/CkMeans1D.cs
===================================================================
--- branches/PerformanceComparison/HeuristicLab.Analysis/3.3/Clustering/CkMeans1D.cs	(revision 13794)
+++ branches/PerformanceComparison/HeuristicLab.Analysis/3.3/Clustering/CkMeans1D.cs	(revision 13794)
@@ -0,0 +1,128 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace HeuristicLab.Analysis {
+  /// <summary>
+  /// Implements the Ckmeans.1d.dp method. It is described in the paper:
+  /// Haizhou Wang and Mingzhou Song. 2011.
+  /// Ckmeans.1d.dp: Optimal k-means Clustering in One Dimension by Dynamic Programming
+  /// The R Journal Vol. 3/2, pp. 29-33.
+  /// available at https://journal.r-project.org/archive/2011-2/RJournal_2011-2_Wang+Song.pdf
+  /// </summary>
+  public class CkMeans1D {
+    /// <summary>
+    /// Clusters the 1-dimensional data given in <paramref name="estimations"/>.
+    /// </summary>
+    /// <param name="estimations">The 1-dimensional data that should be clustered.</param>
+    /// <param name="k">The maximum number of clusters.</param>
+    /// <param name="clusterValues">A vector of the same length as estimations that assigns to each point a cluster id.</param>
+    /// <returns>A sorted list of cluster centroids and corresponding cluster ids.</returns>
+    public static SortedList<double, int> Cluster(double[] estimations, int k, out int[] clusterValues) {
+      int nPoints = estimations.Length;
+      var distinct = estimations.Distinct().OrderBy(x => x).ToArray();
+      var max = distinct.Max();
+      if (distinct.Length <= k) {
+        var dict = distinct.Select((v, i) => new { Index = i, Value = v }).ToDictionary(x => x.Value, y => y.Index);
+        for (int i = distinct.Length; i < k; i++)
+          dict.Add(max + i - distinct.Length + 1, i);
+
+        clusterValues = new int[nPoints];
+        for (int i = 0; i < nPoints; i++)
+          if (!dict.ContainsKey(estimations[i])) clusterValues[i] = 0;
+          else clusterValues[i] = dict[estimations[i]];
+
+        return new SortedList<double, int>(dict);
+      }
+
+      var n = distinct.Length;
+      var D = new double[n, k];
+      var B = new int[n, k];
+
+      for (int m = 0; m < k; m++) {
+        for (int j = m; j <= n - k + m; j++) {
+          if (m == 0)
+            D[j, m] = SumOfSquaredDistances(distinct, 0, j + 1);
+          else {
+            var minD = double.MaxValue;
+            var minI = 0;
+            for (int i = 1; i <= j; i++) {
+              var d = D[i - 1, m - 1] + SumOfSquaredDistances(distinct, i, j + 1);
+              if (d < minD) {
+                minD = d;
+                minI = i;
+              }
+            }
+            D[j, m] = minD;
+            B[j, m] = minI;
+          }
+        }
+      }
+
+      var centers = new SortedList<double, int>();
+      var upper = B[n - 1, k - 1];
+      var c = Mean(distinct, upper, n);
+      centers.Add(c, k - 1);
+      for (int i = k - 2; i >= 0; i--) {
+        var lower = B[upper - 1, i];
+        var c2 = Mean(distinct, lower, upper);
+        centers.Add(c2, i);
+        upper = lower;
+      }
+
+      clusterValues = new int[nPoints];
+      for (int i = 0; i < estimations.Length; i++) {
+        clusterValues[i] = centers.MinItems(x => Math.Abs(estimations[i] - x.Key)).First().Value;
+      }
+
+      return centers;
+    }
+
+    private static double SumOfSquaredDistances(double[] x, int start, int end) {
+      if (start == end) throw new InvalidOperationException();
+      if (start + 1 == end) return 0.0;
+      double mean = 0.0;
+      for (int i = start; i < end; i++) {
+        mean += x[i];
+      }
+      mean /= (end - start);
+      var sum = 0.0;
+      for (int i = start; i < end; i++) {
+        sum += (x[i] - mean) * (x[i] - mean);
+      }
+      return sum;
+    }
+
+    private static double Mean(double[] x, int start, int end) {
+      if (start == end) throw new InvalidOperationException();
+      double mean = 0.0;
+      for (int i = start; i < end; i++) {
+        mean += x[i];
+      }
+      mean /= (end - start);
+      return mean;
+    }
+  }
+}
Index: branches/PerformanceComparison/HeuristicLab.Analysis/3.3/Clustering/ClusterHelper.cs
===================================================================
--- branches/PerformanceComparison/HeuristicLab.Analysis/3.3/Clustering/ClusterHelper.cs	(revision 13794)
+++ branches/PerformanceComparison/HeuristicLab.Analysis/3.3/Clustering/ClusterHelper.cs	(revision 13794)
@@ -0,0 +1,88 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace HeuristicLab.Analysis {
+  public class ClusteringHelper<T> {
+    private readonly int k;
+
+    private List<KeyValuePair<T, double>> instances;
+    private List<KeyValuePair<T, double>> excluded;
+    private int[] clusterValues;
+
+    private ClusteringHelper(int K) {
+      this.k = K;
+    }
+
+    /// <summary>
+    /// Helps in clustering data which is available as key-value pairs.
+    /// It is possible to specify an exclude function to omit certain points from clustering.
+    /// These points will be assigned cluster with id k. All other points will be assigned a
+    /// cluster id in the range [0;k-1].
+    /// </summary>
+    /// <param name="k">The maximum number of clusters that should be created.</param>
+    /// <param name="values">The data which links a certain item with a double value that is to be clustered.</param>
+    /// <param name="excludeFunc">The function that allows excluding certing data points which will receive cluster id k.</param>
+    /// <returns>A reference to the helper class to allow fluent calls.</returns>
+    public static ClusteringHelper<T> Cluster(int k, IEnumerable<KeyValuePair<T, double>> values, Func<KeyValuePair<T, double>, bool> excludeFunc = null) {
+      if (excludeFunc == null) excludeFunc = _ => false;
+      var helper = new ClusteringHelper<T>(k);
+      helper.Initialize(values, excludeFunc);
+      if (helper.instances.Count == 0) 
+        helper.clusterValues = new int[0];
+      else CkMeans1D.Cluster(helper.instances.Select(x => x.Value).ToArray(), k, out helper.clusterValues);
+      return helper;
+    }
+
+    /// <summary>
+    /// Returns the clustered data by 
+    /// </summary>
+    /// <returns></returns>
+    public IEnumerable<KeyValuePair<T, Tuple<double, int>>> GetByInstance() {
+      return GetClustered().Select(x => new KeyValuePair<T, Tuple<double, int>>(x.Item1.Key, Tuple.Create(x.Item1.Value, x.Item2)));
+    }
+
+    public IEnumerable<KeyValuePair<int, List<KeyValuePair<T, double>>>> GetByCluster() {
+      return GetClustered().GroupBy(x => x.Item2)
+        .Select(x => new KeyValuePair<int, List<KeyValuePair<T, double>>>(x.Key, x.Select(y => y.Item1).ToList()));
+    }
+
+    private void Initialize(IEnumerable<KeyValuePair<T, double>> values, Func<KeyValuePair<T, double>, bool> excludeFunc) {
+      instances = new List<KeyValuePair<T, double>>();
+      excluded = new List<KeyValuePair<T, double>>();
+      foreach (var v in values) {
+        if (!excludeFunc(v)) instances.Add(v);
+        else excluded.Add(v);
+      }
+    }
+
+    private IEnumerable<Tuple<KeyValuePair<T, double>, int>> GetClustered() {
+      for (var i = 0; i < clusterValues.Length; i++) {
+        yield return Tuple.Create(instances[i], clusterValues[i]);
+      }
+      foreach (var ex in excluded)
+        yield return Tuple.Create(ex, k);
+    }
+  }
+}
Index: branches/PerformanceComparison/HeuristicLab.Analysis/3.3/HeuristicLab.Analysis-3.3.csproj
===================================================================
--- branches/PerformanceComparison/HeuristicLab.Analysis/3.3/HeuristicLab.Analysis-3.3.csproj	(revision 13792)
+++ branches/PerformanceComparison/HeuristicLab.Analysis/3.3/HeuristicLab.Analysis-3.3.csproj	(revision 13794)
@@ -187,4 +187,8 @@
     <Compile Include="BestNScopesSolutionAnalyzer.cs" />
     <Compile Include="BestScopeSolutionAnalyzer.cs" />
+    <Compile Include="Clustering\CkMeans1D.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="Clustering\ClusterHelper.cs" />
     <Compile Include="DataVisualization\IndexedDataRow.cs" />
     <Compile Include="DataVisualization\IndexedDataTable.cs" />
Index: branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Interfaces/IRecommendationModel.cs
===================================================================
--- branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Interfaces/IRecommendationModel.cs	(revision 13792)
+++ branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Interfaces/IRecommendationModel.cs	(revision 13794)
@@ -22,10 +22,9 @@
 using HeuristicLab.Common;
 using HeuristicLab.Optimization;
-using System;
 using System.Collections.Generic;
 
 namespace HeuristicLab.OptimizationExpertSystem.Common {
   public interface IRecommendationModel : IContent {
-    IEnumerable<Tuple<IAlgorithm, double>> GetRanking(IRun problemInstance);
+    IEnumerable<KeyValuePair<IAlgorithm, double>> GetRanking(IRun problemInstance);
   }
 }
Index: branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/KnowledgeCenter.cs
===================================================================
--- branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/KnowledgeCenter.cs	(revision 13792)
+++ branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/KnowledgeCenter.cs	(revision 13794)
@@ -718,32 +718,13 @@
         result[problemMap[pr.Key]] = new Dictionary<long, int>();
 
-        var indexMap = new BidirectionalDictionary<long, int>();
-        var perf = new Dictionary<long, double>();
-        var idx = 0;
-        foreach (var kvp in pr.GroupBy(x => algorithmId2RunMapping.GetBySecond(x).Single())) {
-          var ert = ExpectedRuntimeHelper.CalculateErt(kvp.ToList(), "QualityPerEvaluations", GetTarget(bkq, max), max).ExpectedRuntime;
-          if (double.IsNaN(ert)) {
-            // algorithm can't solve that instance with the given target
-            // these are put into their own additional class
-            result[problemMap[pr.Key]][kvp.Key] = nClasses;
-            continue;
-          }
-          indexMap.Add(kvp.Key, idx);
-          perf[kvp.Key] = Math.Log10(ert);
-          idx++;
-        }
-        if (perf.Count == 0) continue;
-        
-        var points = perf.OrderBy(x => indexMap.GetByFirst(x.Key)).Select(x => x.Value).ToArray();
-        int[] clusters;
-        Ckmeans1dClusters(points, nClasses, out clusters);
-        var ranks = clusters.Select((c, i) => new { Cluster = c, Perf = perf[indexMap.GetBySecond(i)] })
-                            .GroupBy(x => x.Cluster, x => x.Perf)
-                            .Select(x => new { Cluster = x.Key, AvgPerf = x.Average() })
-                            .OrderBy(x => x.AvgPerf)
-                            .Select((c, i) => new { Cluster = c.Cluster, Rank = i })
-                            .ToDictionary(x => x.Cluster, x => x.Rank);
-        for (var i = 0; i < clusters.Length; i++)
-          result[problemMap[pr.Key]][indexMap.GetBySecond(i)] = ranks[clusters[i]];
+        var values = pr.GroupBy(x => algorithmId2RunMapping.GetBySecond(x).Single())
+                       .ToDictionary(x => x.Key, x => ExpectedRuntimeHelper.CalculateErt(x.ToList(), "QualityPerEvaluations", GetTarget(bkq, max), max).ExpectedRuntime);
+        var ranks = ClusteringHelper<long>.Cluster(nClasses, values, x => double.IsNaN(x.Value))
+          .GetByCluster().ToList();
+        var minRank = ranks.Min(x => x.Key);
+        foreach (var c in ranks) {
+          foreach (var a in c.Value)
+            result[problemMap[pr.Key]][a.Key] = c.Key == nClasses ? c.Key : c.Key - minRank;
+        }
       }
       return result;
@@ -772,91 +753,5 @@
     }
 
-    // implement further classes and methods
-    private static SortedList<double, int> Ckmeans1dClusters(double[] estimations, int k, out int[] clusterValues) {
-      int nPoints = estimations.Length;
-      var distinct = estimations.Distinct().OrderBy(x => x).ToArray();
-      var max = distinct.Max();
-      if (distinct.Length <= k) {
-        var dict = distinct.Select((v, i) => new { Index = i, Value = v }).ToDictionary(x => x.Value, y => y.Index);
-        for (int i = distinct.Length; i < k; i++)
-          dict.Add(max + i - distinct.Length + 1, i);
-
-        clusterValues = new int[nPoints];
-        for (int i = 0; i < nPoints; i++)
-          if (!dict.ContainsKey(estimations[i])) clusterValues[i] = 0;
-          else clusterValues[i] = dict[estimations[i]];
-
-        return new SortedList<double, int>(dict);
-      }
-
-      var n = distinct.Length;
-      var D = new double[n, k];
-      var B = new int[n, k];
-
-      for (int m = 0; m < k; m++) {
-        for (int j = m; j <= n - k + m; j++) {
-          if (m == 0)
-            D[j, m] = SumOfSquaredDistances(distinct, 0, j + 1);
-          else {
-            var minD = double.MaxValue;
-            var minI = 0;
-            for (int i = 1; i <= j; i++) {
-              var d = D[i - 1, m - 1] + SumOfSquaredDistances(distinct, i, j + 1);
-              if (d < minD) {
-                minD = d;
-                minI = i;
-              }
-            }
-            D[j, m] = minD;
-            B[j, m] = minI;
-          }
-        }
-      }
-
-      var centers = new SortedList<double, int>();
-      var upper = B[n - 1, k - 1];
-      var c = Mean(distinct, upper, n);
-      centers.Add(c, k - 1);
-      for (int i = k - 2; i >= 0; i--) {
-        var lower = B[upper - 1, i];
-        var c2 = Mean(distinct, lower, upper);
-        centers.Add(c2, i);
-        upper = lower;
-      }
-
-      clusterValues = new int[nPoints];
-      for (int i = 0; i < estimations.Length; i++) {
-        clusterValues[i] = centers.MinItems(x => Math.Abs(estimations[i] - x.Key)).First().Value;
-      }
-
-      return centers;
-    }
-
-    private static double SumOfSquaredDistances(double[] x, int start, int end) {
-      if (start == end) throw new InvalidOperationException();
-      if (start + 1 == end) return 0.0;
-      double mean = 0.0;
-      for (int i = start; i < end; i++) {
-        mean += x[i];
-      }
-      mean /= (end - start);
-      var sum = 0.0;
-      for (int i = start; i < end; i++) {
-        sum += (x[i] - mean) * (x[i] - mean);
-      }
-      return sum;
-    }
-
-    private static double Mean(double[] x, int start, int end) {
-      if (start == end) throw new InvalidOperationException();
-      double mean = 0.0;
-      for (int i = start; i < end; i++) {
-        mean += x[i];
-      }
-      mean /= (end - start);
-      return mean;
-    }
-
-    public IEnumerable<Tuple<IAlgorithm, double>> GetAlgorithmInstanceRanking() {
+    public IEnumerable<KeyValuePair<IAlgorithm, double>> GetAlgorithmInstanceRanking() {
       return RecommendationModel.GetRanking(ProblemInstances.Single(IsCurrentInstance));
     }
Index: branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/FixedRankModel.cs
===================================================================
--- branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/FixedRankModel.cs	(revision 13792)
+++ branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/FixedRankModel.cs	(revision 13794)
@@ -21,22 +21,21 @@
 
 using HeuristicLab.Optimization;
-using System;
 using System.Collections.Generic;
 
 namespace HeuristicLab.OptimizationExpertSystem.Common {
   public class FixedRankModel : IRecommendationModel {
-    private readonly List<Tuple<IAlgorithm, double>> ranking;
+    private readonly List<KeyValuePair<IAlgorithm, double>> ranking;
 
-    public FixedRankModel(IEnumerable<Tuple<IAlgorithm, double>> ranking) {
-      this.ranking = new List<Tuple<IAlgorithm, double>>();
+    public FixedRankModel(IEnumerable<KeyValuePair<IAlgorithm, double>> ranking) {
+      this.ranking = new List<KeyValuePair<IAlgorithm, double>>();
       foreach (var r in ranking) this.ranking.Add(r);
     }
  
-    public IEnumerable<Tuple<IAlgorithm, double>> GetRanking(IRun problemInstance) {
+    public IEnumerable<KeyValuePair<IAlgorithm, double>> GetRanking(IRun problemInstance) {
       return ranking;
     }
 
     public static FixedRankModel GetEmpty() {
-      return new FixedRankModel(new Tuple<IAlgorithm, double>[0]);
+      return new FixedRankModel(new KeyValuePair<IAlgorithm, double>[0]);
     }
   }
Index: branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/KNearestNeighborModel.cs
===================================================================
--- branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/KNearestNeighborModel.cs	(revision 13792)
+++ branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/KNearestNeighborModel.cs	(revision 13794)
@@ -22,5 +22,4 @@
 using HeuristicLab.Collections;
 using HeuristicLab.Optimization;
-using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -46,5 +45,5 @@
     }
 
-    public IEnumerable<Tuple<IAlgorithm, double>> GetRanking(IRun problemInstance) {
+    public IEnumerable<KeyValuePair<IAlgorithm, double>> GetRanking(IRun problemInstance) {
       var features = KnowledgeCenter.GetFeatures(performance.Keys.OrderBy(problemInstanceMap.GetBySecond).ToArray(), characteristics, medianValues);
       var feature = KnowledgeCenter.GetFeatures(new [] { problemInstance }, characteristics, medianValues)[0];
@@ -70,5 +69,5 @@
       return performances.Select(x => new { Alg = x.Key, Perf = x.Value.Average() })
                          .OrderBy(x => x.Perf)
-                         .Select(x => Tuple.Create(x.Alg, x.Perf));
+                         .Select(x => new KeyValuePair<IAlgorithm, double>(x.Alg, x.Perf));
     }
   }
Index: branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/OverallBestRecommender.cs
===================================================================
--- branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/OverallBestRecommender.cs	(revision 13792)
+++ branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/OverallBestRecommender.cs	(revision 13794)
@@ -49,5 +49,5 @@
 
     public IRecommendationModel TrainModel(IRun[] problemInstances, KnowledgeCenter kc, string[] characteristics) {
-      var instances = new List<Tuple<IAlgorithm, double>>();
+      var instances = new List<KeyValuePair<IAlgorithm, double>>();
       foreach (var relevantRuns in kc.GetKnowledgeBaseByAlgorithm()) {
         var algorithm = relevantRuns.Key;
@@ -60,5 +60,6 @@
         var count = 0;
         foreach (var problemRuns in relevantRuns.Value.GroupBy(x => ((StringValue)x.Parameters["Problem Name"]).Value)) {
-          var bkq = pis[problemRuns.Key];
+          double bkq;
+          if (!pis.TryGetValue(problemRuns.Key, out bkq)) continue;
           var ert = ExpectedRuntimeHelper.CalculateErt(problemRuns.ToList(), "QualityPerEvaluations", kc.GetTarget(bkq, kc.Maximization), kc.Maximization).ExpectedRuntime;
           if (double.IsNaN(ert)) ert = int.MaxValue;
@@ -67,8 +68,8 @@
         }
         avgERT /= count;
-        instances.Add(Tuple.Create((IAlgorithm)algorithm.Clone(), avgERT));
+        instances.Add(new KeyValuePair<IAlgorithm, double>(algorithm, avgERT));
       }
 
-      return new FixedRankModel(instances.OrderBy(x => x.Item2));
+      return new FixedRankModel(instances.OrderBy(x => x.Value));
     }
   }
Index: branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.Designer.cs
===================================================================
--- branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.Designer.cs	(revision 13792)
+++ branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.Designer.cs	(revision 13794)
@@ -55,7 +55,14 @@
       this.parameterCollectionView = new HeuristicLab.Core.Views.ParameterCollectionView();
       this.crossvalidationTabPage = new System.Windows.Forms.TabPage();
+      this.kendallsTauLabel = new System.Windows.Forms.Label();
+      this.absoluteErrorLabel = new System.Windows.Forms.Label();
+      this.kendallsTauView = new HeuristicLab.Data.Views.StringConvertibleValueView();
+      this.absoluteErrorView = new HeuristicLab.Data.Views.StringConvertibleValueView();
+      this.confusionMatrixView = new HeuristicLab.Data.Views.StringConvertibleMatrixView();
       this.xValidateButton = new System.Windows.Forms.Button();
       this.minTargetView = new HeuristicLab.Data.Views.StringConvertibleValueView();
       this.minimumTargetLabel = new System.Windows.Forms.Label();
+      this.actualLabel = new System.Windows.Forms.Label();
+      this.predictedLabel = new System.Windows.Forms.Label();
       this.tabControl.SuspendLayout();
       this.characteristicsTabPage.SuspendLayout();
@@ -75,6 +82,4 @@
       // recommenderComboBox
       // 
-      this.recommenderComboBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left) 
-            | System.Windows.Forms.AnchorStyles.Right)));
       this.recommenderComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
       this.recommenderComboBox.FormattingEnabled = true;
@@ -87,5 +92,4 @@
       // recommendStartButton
       // 
-      this.recommendStartButton.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
       this.recommendStartButton.Location = new System.Drawing.Point(442, 1);
       this.recommendStartButton.Name = "recommendStartButton";
@@ -107,5 +111,5 @@
       this.tabControl.Name = "tabControl";
       this.tabControl.SelectedIndex = 0;
-      this.tabControl.Size = new System.Drawing.Size(700, 350);
+      this.tabControl.Size = new System.Drawing.Size(817, 509);
       this.tabControl.TabIndex = 14;
       // 
@@ -174,12 +178,73 @@
       // crossvalidationTabPage
       // 
+      this.crossvalidationTabPage.Controls.Add(this.kendallsTauLabel);
+      this.crossvalidationTabPage.Controls.Add(this.predictedLabel);
+      this.crossvalidationTabPage.Controls.Add(this.actualLabel);
+      this.crossvalidationTabPage.Controls.Add(this.absoluteErrorLabel);
+      this.crossvalidationTabPage.Controls.Add(this.kendallsTauView);
+      this.crossvalidationTabPage.Controls.Add(this.absoluteErrorView);
+      this.crossvalidationTabPage.Controls.Add(this.confusionMatrixView);
       this.crossvalidationTabPage.Controls.Add(this.xValidateButton);
       this.crossvalidationTabPage.Location = new System.Drawing.Point(4, 22);
       this.crossvalidationTabPage.Name = "crossvalidationTabPage";
       this.crossvalidationTabPage.Padding = new System.Windows.Forms.Padding(3);
-      this.crossvalidationTabPage.Size = new System.Drawing.Size(692, 324);
+      this.crossvalidationTabPage.Size = new System.Drawing.Size(809, 483);
       this.crossvalidationTabPage.TabIndex = 1;
       this.crossvalidationTabPage.Text = "Crossvalidation";
       this.crossvalidationTabPage.UseVisualStyleBackColor = true;
+      // 
+      // kendallsTauLabel
+      // 
+      this.kendallsTauLabel.AutoSize = true;
+      this.kendallsTauLabel.Location = new System.Drawing.Point(536, 110);
+      this.kendallsTauLabel.Margin = new System.Windows.Forms.Padding(3);
+      this.kendallsTauLabel.Name = "kendallsTauLabel";
+      this.kendallsTauLabel.Size = new System.Drawing.Size(104, 13);
+      this.kendallsTauLabel.TabIndex = 3;
+      this.kendallsTauLabel.Text = "Mean Kendall\'s Tau:";
+      // 
+      // absoluteErrorLabel
+      // 
+      this.absoluteErrorLabel.AutoSize = true;
+      this.absoluteErrorLabel.Location = new System.Drawing.Point(536, 52);
+      this.absoluteErrorLabel.Margin = new System.Windows.Forms.Padding(3);
+      this.absoluteErrorLabel.Name = "absoluteErrorLabel";
+      this.absoluteErrorLabel.Size = new System.Drawing.Size(106, 13);
+      this.absoluteErrorLabel.TabIndex = 3;
+      this.absoluteErrorLabel.Text = "Mean Absolute Error:";
+      // 
+      // kendallsTauView
+      // 
+      this.kendallsTauView.Caption = "StringConvertibleValue View";
+      this.kendallsTauView.Content = null;
+      this.kendallsTauView.LabelVisible = false;
+      this.kendallsTauView.Location = new System.Drawing.Point(520, 129);
+      this.kendallsTauView.Name = "kendallsTauView";
+      this.kendallsTauView.ReadOnly = true;
+      this.kendallsTauView.Size = new System.Drawing.Size(194, 21);
+      this.kendallsTauView.TabIndex = 2;
+      // 
+      // absoluteErrorView
+      // 
+      this.absoluteErrorView.Caption = "StringConvertibleValue View";
+      this.absoluteErrorView.Content = null;
+      this.absoluteErrorView.LabelVisible = false;
+      this.absoluteErrorView.Location = new System.Drawing.Point(520, 71);
+      this.absoluteErrorView.Name = "absoluteErrorView";
+      this.absoluteErrorView.ReadOnly = true;
+      this.absoluteErrorView.Size = new System.Drawing.Size(194, 21);
+      this.absoluteErrorView.TabIndex = 2;
+      // 
+      // confusionMatrixView
+      // 
+      this.confusionMatrixView.Caption = "StringConvertibleMatrix View";
+      this.confusionMatrixView.Content = null;
+      this.confusionMatrixView.Location = new System.Drawing.Point(37, 71);
+      this.confusionMatrixView.Name = "confusionMatrixView";
+      this.confusionMatrixView.ReadOnly = true;
+      this.confusionMatrixView.ShowRowsAndColumnsTextBox = false;
+      this.confusionMatrixView.ShowStatisticalInformation = false;
+      this.confusionMatrixView.Size = new System.Drawing.Size(477, 363);
+      this.confusionMatrixView.TabIndex = 1;
       // 
       // xValidateButton
@@ -212,4 +277,26 @@
       this.minimumTargetLabel.TabIndex = 17;
       this.minimumTargetLabel.Text = "Target:";
+      // 
+      // actualLabel
+      // 
+      this.actualLabel.AutoSize = true;
+      this.actualLabel.Font = new System.Drawing.Font("Microsoft Sans Serif", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
+      this.actualLabel.Location = new System.Drawing.Point(16, 210);
+      this.actualLabel.Margin = new System.Windows.Forms.Padding(3);
+      this.actualLabel.Name = "actualLabel";
+      this.actualLabel.Size = new System.Drawing.Size(15, 78);
+      this.actualLabel.TabIndex = 3;
+      this.actualLabel.Text = "A\r\nC\r\nT\r\nU\r\nA\r\nL";
+      // 
+      // predictedLabel
+      // 
+      this.predictedLabel.AutoSize = true;
+      this.predictedLabel.Font = new System.Drawing.Font("Microsoft Sans Serif", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
+      this.predictedLabel.Location = new System.Drawing.Point(241, 52);
+      this.predictedLabel.Margin = new System.Windows.Forms.Padding(3);
+      this.predictedLabel.Name = "predictedLabel";
+      this.predictedLabel.Size = new System.Drawing.Size(69, 13);
+      this.predictedLabel.TabIndex = 3;
+      this.predictedLabel.Text = "PREDICTED";
       // 
       // PerformanceModelingView
@@ -224,8 +311,10 @@
       this.Controls.Add(this.recommendStartButton);
       this.Name = "PerformanceModelingView";
+      this.Size = new System.Drawing.Size(817, 539);
       this.tabControl.ResumeLayout(false);
       this.characteristicsTabPage.ResumeLayout(false);
       this.parametersTabPage.ResumeLayout(false);
       this.crossvalidationTabPage.ResumeLayout(false);
+      this.crossvalidationTabPage.PerformLayout();
       this.ResumeLayout(false);
       this.PerformLayout();
@@ -248,4 +337,11 @@
     private Data.Views.StringConvertibleValueView minTargetView;
     private System.Windows.Forms.Label minimumTargetLabel;
+    private System.Windows.Forms.Label kendallsTauLabel;
+    private System.Windows.Forms.Label absoluteErrorLabel;
+    private Data.Views.StringConvertibleValueView kendallsTauView;
+    private Data.Views.StringConvertibleValueView absoluteErrorView;
+    private Data.Views.StringConvertibleMatrixView confusionMatrixView;
+    private System.Windows.Forms.Label predictedLabel;
+    private System.Windows.Forms.Label actualLabel;
   }
 }
Index: branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.cs
===================================================================
--- branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.cs	(revision 13792)
+++ branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.cs	(revision 13794)
@@ -20,4 +20,5 @@
 #endregion
 
+using HeuristicLab.Analysis;
 using HeuristicLab.Common.Resources;
 using HeuristicLab.Core;
@@ -134,31 +135,76 @@
       var trainingSet = Content.ProblemInstances.Where(x => !Content.IsCurrentInstance(x)).ToArray();
 
+      var absErr = 0.0;
+      var confMatrix = new int[6, 6];
+      var tau = 0.0;
       // leave one out crossvalidation
       foreach (var pi in trainingSet) {
         var model = recommender.TrainModel(trainingSet.Where(x => x != pi).ToArray(), Content, features);
-        var ranking = model.GetRanking(pi).ToList();
-        var realPerformance = Content.GetAlgorithmPerformance(pi);
-        // absolute error predicting ert
-        // confusion matrix predicting class
-        // Kendall's tau
+        var predicted = model.GetRanking(pi).ToDictionary(x => x.Key, x => x.Value);
+        var observed = Content.GetAlgorithmPerformance(pi);
+        absErr += AbsoluteError(observed, predicted);
+        var confMat = ConfusionMatrix(observed, predicted);
+        for (var i = 0; i < confMat.GetLength(0); i++)
+          for (var j = 0; j < confMat.GetLength(1); j++)
+            confMatrix[i, j] += confMat[i, j];
+        tau += KendallsTau(observed, predicted);
         // average NCDG ... relevance determined by clustering (unsuccessful algorithms being penalized with negative relevance)
         // mean reciprocal rank
         // optional: expected reciprocal rank
       }
-    }
-
-    private static double AbsoluteError(Dictionary<IAlgorithm, double> performance, List<Tuple<IAlgorithm, double>> ranking) {
+      absErr /= trainingSet.Length;
+      tau /= trainingSet.Length;
+
+      confusionMatrixView.Content = new IntMatrix(confMatrix);
+      absoluteErrorView.Content = new DoubleValue(absErr);
+      kendallsTauView.Content = new DoubleValue(tau);
+    }
+
+    private static double AbsoluteError(Dictionary<IAlgorithm, double> performance, Dictionary<IAlgorithm, double> ranking) {
       var error = 0.0;
       foreach (var tuple in ranking) {
         double actual;
-        if (!performance.TryGetValue(tuple.Item1, out actual)) continue;
-        error += Math.Abs(actual - tuple.Item2);
+        if (!performance.TryGetValue(tuple.Key, out actual)) continue;
+        if (double.IsNaN(actual)) actual = int.MaxValue;
+        error += Math.Abs(actual - tuple.Value);
       }
       return error;
     }
 
-    private static int[,] ConfusionMatrix(Dictionary<IAlgorithm, double> performance, List<Tuple<IAlgorithm, double>> ranking) {
-      var confMatrix = new int[5,5];
+    private static int[,] ConfusionMatrix(Dictionary<IAlgorithm, double> performance, Dictionary<IAlgorithm, double> ranking) {
+      var confMatrix = new int[6,6];
+      var clusteredPerformance = ClusteringHelper<IAlgorithm>.Cluster(5, performance, a => double.IsNaN(a.Value)).GetByInstance().ToDictionary(x => x.Key, x => x.Value.Item2);
+      var clusteredRanks = ClusteringHelper<IAlgorithm>.Cluster(5, ranking, x => double.IsNaN(x.Value) || x.Value >= int.MaxValue).GetByInstance().ToDictionary(x => x.Key, x => x.Value.Item2);
+      foreach (var cr in clusteredRanks) {
+        int realRank;
+        if (!clusteredPerformance.TryGetValue(cr.Key, out realRank)) continue;
+        confMatrix[realRank, cr.Value]++;
+      }
       return confMatrix;
+    }
+
+    private static double KendallsTau(Dictionary<IAlgorithm, double> performance, Dictionary<IAlgorithm, double> ranking) {
+      var commonKeys = performance.Keys.Intersect(ranking.Keys).ToList();
+      var n = commonKeys.Count;
+      if (n == 0) return 0.0;
+
+      var actualRanks = commonKeys.Select((x, i) => new { Alg = x, Index = i, Perf = performance[x] })
+                                  .OrderBy(x => x.Perf).Select((x, i) => new { Index = x.Index, Rank = i })
+                                  .OrderBy(x => x.Index).Select(x => x.Rank).ToList();
+      var predictedRanks = commonKeys.Select((x, i) => new { Alg = x, Index = i, Perf = ranking[x] })
+                                     .OrderBy(x => x.Perf).Select((x, i) => new { Index = x.Index, Rank = i })
+                                     .OrderBy(x => x.Index).Select(x => x.Rank).ToList();
+      
+      var paired = actualRanks.Zip(predictedRanks, (a, p) => new { Actual = a, Predicted = p }).ToList();
+      var concordant = 0;
+      var discordant = 0;
+      for (var i = 0; i < paired.Count - 1; i++)
+        for (var j = 0; j < paired.Count; j++) {
+          if (paired[i].Actual > paired[j].Actual && paired[i].Predicted > paired[j].Predicted
+            || paired[i].Actual < paired[j].Actual && paired[i].Predicted < paired[j].Predicted)
+            concordant++;
+          else discordant++;
+        }
+      return (2.0 * (concordant - discordant)) / (n * (n - 1));
     }
     #endregion
Index: branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/SolverView.cs
===================================================================
--- branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/SolverView.cs	(revision 13792)
+++ branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/SolverView.cs	(revision 13794)
@@ -28,4 +28,5 @@
 using HeuristicLab.OptimizationExpertSystem.Common;
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Windows.Forms;
@@ -96,5 +97,5 @@
       for (var i = 0; i < ranking.Count; i++) {
         suggestedInstancesComboBox.Items.Add(new AlgorithmInstanceItem(ranking[i]));
-        if (prevSelection == null || ranking[i].Item1.Name == prevSelection.Item.Item1.Name)
+        if (prevSelection == null || ranking[i].Key.Name == prevSelection.Item.Key.Name)
           prevNewIndex = prevSelection == null ? 0 : i;
       }
@@ -119,10 +120,10 @@
     private void AlgorithmStartButtonOnClick(object sender, EventArgs e) {
       if (suggestedInstancesComboBox.SelectedIndex >= 0)
-        Content.StartAlgorithmAsync(Content.AlgorithmInstances.Select((a, i) => new { Alg = a, Index = i}).Single(x => x.Alg.Name == ((AlgorithmInstanceItem)suggestedInstancesComboBox.SelectedItem).Item.Item1.Name).Index);
+        Content.StartAlgorithmAsync(Content.AlgorithmInstances.Select((a, i) => new { Alg = a, Index = i}).Single(x => x.Alg.Name == ((AlgorithmInstanceItem)suggestedInstancesComboBox.SelectedItem).Item.Key.Name).Index);
     }
 
     private void AlgorithmCloneButtonOnClick(object sender, EventArgs e) {
       if (suggestedInstancesComboBox.SelectedIndex >= 0)
-        MainForm.ShowContent((IAlgorithm)Content.AlgorithmInstances[Content.AlgorithmInstances.Select((a, i) => new { Alg = a, Index = i }).Single(x => x.Alg.Name == ((AlgorithmInstanceItem)suggestedInstancesComboBox.SelectedItem).Item.Item1.Name).Index].Clone());
+        MainForm.ShowContent((IAlgorithm)Content.AlgorithmInstances[Content.AlgorithmInstances.Select((a, i) => new { Alg = a, Index = i }).Single(x => x.Alg.Name == ((AlgorithmInstanceItem)suggestedInstancesComboBox.SelectedItem).Item.Key.Name).Index].Clone());
     }
 
@@ -130,5 +131,5 @@
       if (InvokeRequired) { Invoke((Action<object, EventArgs>)SuggestedInstancesComboBoxOnSelectedIndexChanged, sender, e); return; }
       if (suggestedInstancesComboBox.SelectedIndex >= 0) {
-        var alg = Content.AlgorithmInstances[Content.AlgorithmInstances.Select((a, i) => new { Alg = a, Index = i }).Single(x => x.Alg.Name == ((AlgorithmInstanceItem)suggestedInstancesComboBox.SelectedItem).Item.Item1.Name).Index];
+        var alg = Content.AlgorithmInstances[Content.AlgorithmInstances.Select((a, i) => new { Alg = a, Index = i }).Single(x => x.Alg.Name == ((AlgorithmInstanceItem)suggestedInstancesComboBox.SelectedItem).Item.Key.Name).Index];
         solverParametersView.Content = alg.Parameters;
         var engineAlg = alg as EngineAlgorithm;
@@ -145,13 +146,13 @@
     #region Helper Classes and Methods
     private class AlgorithmInstanceItem {
-      private readonly Tuple<IAlgorithm, double> item;
-      public Tuple<IAlgorithm, double> Item { get { return item; } }
+      private readonly KeyValuePair<IAlgorithm, double> item;
+      public KeyValuePair<IAlgorithm, double> Item { get { return item; } }
 
-      public AlgorithmInstanceItem(Tuple<IAlgorithm, double> item) {
+      public AlgorithmInstanceItem(KeyValuePair<IAlgorithm, double> item) {
         this.item = item;
       }
 
       public override string ToString() {
-        return item.Item2.ToString("N0") + ": " + item.Item1.Name;
+        return item.Value.ToString("N0") + ": " + item.Key.Name;
       }
     }
