Index: branches/PerformanceComparison/ProblemInstanceIdentifier/InstanceDescriptor.cs
===================================================================
--- branches/PerformanceComparison/ProblemInstanceIdentifier/InstanceDescriptor.cs	(revision 14690)
+++ branches/PerformanceComparison/ProblemInstanceIdentifier/InstanceDescriptor.cs	(revision 14691)
@@ -46,12 +46,12 @@
 
     public static InstanceDescriptor FromPaths(QuadraticAssignmentProblem qap, List<List<Tuple<Permutation, double>>> trajectories) {
-      var features = QAPDirectedWalk.Calculate(trajectories).ToDictionary(x => x.Name, x => x.Value);
-      
+      var result = PermutationPathAnalysis.GetCharacteristics(trajectories);
+
       return new InstanceDescriptor() {
         Name = qap.Name,
         Cls = GetClass(qap.Name),
         Dimension = qap.Weights.Rows,
-        FeatureNames = features.Keys.ToArray(),
-        FeatureValues = features.Values.Select(x => ((DoubleValue)x).Value).ToArray()
+        FeatureNames = result.GetNames(),
+        FeatureValues = result.GetValues()
       };
     }
@@ -87,4 +87,11 @@
     private double[] featureStdevs;
 
+    public IEnumerable<double> GetMeans() {
+      return featureMeans;
+    }
+    public IEnumerable<double> GetStdevs() {
+      return featureStdevs;
+    }
+
     private InstancesStandardizer() { }
 
@@ -101,4 +108,9 @@
       return standardizer;
     }
+    public static InstancesStandardizer CreateAndApply(IList<InstanceDescriptor> instances) {
+      var standardizer = Create(instances);
+      standardizer.Apply(instances);
+      return standardizer;
+    }
 
     public void Apply(IList<InstanceDescriptor> instances) {
Index: branches/PerformanceComparison/ProblemInstanceIdentifier/InstanceExplorer.cs
===================================================================
--- branches/PerformanceComparison/ProblemInstanceIdentifier/InstanceExplorer.cs	(revision 14690)
+++ branches/PerformanceComparison/ProblemInstanceIdentifier/InstanceExplorer.cs	(revision 14691)
@@ -1,4 +1,6 @@
-﻿using System.Linq;
+﻿using System;
+using System.Linq;
 using System.Threading;
+using HeuristicLab.Algorithms.MemPR.Permutation;
 using HeuristicLab.Analysis.FitnessLandscape;
 using HeuristicLab.Data;
@@ -57,5 +59,5 @@
       var walk = new RandomWalk() {
         SeedParameter = { Value = { Value = seed ?? 0 } },
-        SetSeedRandomlyParameter = { Value = { Value = seed.HasValue } },
+        SetSeedRandomlyParameter = { Value = { Value = !seed.HasValue } },
         MaximumIterationsParameter = { Value = { Value = Iterations } },
         RepetitionsParameter = { Value = { Value = 1 } }
@@ -71,3 +73,41 @@
     }
   }
+
+  public class MemPRExplorer : InstanceExplorer {
+    public int Seconds { get; set; }
+
+    public bool IncludeLocalSearch { get; set; }
+
+    public override string Name { get { return "MemPR Explorer"; } }
+    public override int Effort { get { return Seconds; } }
+
+    public MemPRExplorer() {
+
+    }
+
+    public override InstanceDescriptor Explore(QuadraticAssignmentProblem problem, int? seed = null) {
+      var memPr = new PermutationMemPR();
+      memPr.Problem = problem;
+      memPr.Prepare(true);
+      memPr.MaximumExecutionTime = TimeSpan.FromSeconds(Seconds);
+      memPr.SetSeedRandomly = !seed.HasValue;
+      memPr.Seed = seed ?? 0;
+      memPr.StartSync();
+      if (memPr.Context.RelinkedPaths.IsEmpty
+        || IncludeLocalSearch && memPr.Context.LocalSearchPaths.IsEmpty) {
+        Console.Write("{0} not all paths present!", problem.Name);
+        return null;
+      };
+
+      var features = PermutationPathAnalysis.GetCharacteristics(memPr.Context.RelinkedPaths.Paths.ToList());
+      var result = features.GetValues();
+      var resultNames = features.GetNames();
+      if (IncludeLocalSearch) {
+        features = PermutationPathAnalysis.GetCharacteristics(memPr.Context.LocalSearchPaths.Paths.ToList());
+        result = result.Concat(features.GetValues()).ToArray();
+        resultNames = resultNames.Concat(features.GetNames()).ToArray();
+      }
+      return new InstanceDescriptor(problem.Name, InstanceDescriptor.GetClass(problem.Name), problem.Weights.Rows, resultNames, result);
+    }
+  }
 }
Index: branches/PerformanceComparison/ProblemInstanceIdentifier/Program.cs
===================================================================
--- branches/PerformanceComparison/ProblemInstanceIdentifier/Program.cs	(revision 14690)
+++ branches/PerformanceComparison/ProblemInstanceIdentifier/Program.cs	(revision 14691)
@@ -9,4 +9,5 @@
 using HeuristicLab.Problems.Instances.QAPLIB;
 using HeuristicLab.Problems.QuadraticAssignment;
+using HeuristicLab.Random;
 
 namespace ProblemInstanceIdentifier {
@@ -18,4 +19,54 @@
     };
     static void Main(string[] args) {
+      var instances = Get20DifferentClasses();
+      //var instances = GetSomeRandomInstances(50);
+
+      /*var classes = instances.Select(InstanceDescriptor.FromProblemOnly).GroupBy(x => x.Cls);
+      foreach (var cls in classes.OrderBy(x => x.Key)) {
+        Console.WriteLine("{0};{1}", cls.Key, cls.Count());
+      }*/
+
+      var prExplorer = new PathRelinkingExplorer() {
+        LocalOptima = false,
+        Paths = 200
+      };
+      var prLocalExplorer = new PathRelinkingExplorer() {
+        LocalOptima = true,
+        Paths = 200
+      };
+      var memPrExplorer = new MemPRExplorer() {
+        IncludeLocalSearch = false,
+        Seconds = 10
+      };
+
+      var training = GenerateData(instances, prExplorer, parallel:true);
+      var standardizer = InstancesStandardizer.CreateAndApply(training);
+      var test = GenerateData(instances, prExplorer, parallel: false);
+      standardizer.Apply(test);
+      PrintMatchResult(Compare(training, test));
+
+      ExploreMatching(instances, new [] { prLocalExplorer }, new [] { memPrExplorer });
+    }
+
+    private static List<QuadraticAssignmentProblem> GetSomeRandomInstances(int totalInstances) {
+      var sync = new object();
+      var provider = new OneSizeInstanceProvider();
+      var instances = new List<QuadraticAssignmentProblem>();
+      var random = new FastRandom(0);
+      Parallel.ForEach(provider.GetDataDescriptors().Shuffle(random), desc => {
+        var qapData = provider.LoadData(desc);
+        if (qapData.Dimension < 25) return;
+        if (instances.Count >= totalInstances) return;
+        var qap = new QuadraticAssignmentProblem();
+        qap.Load(qapData);
+        lock (sync) {
+          if (instances.Count >= totalInstances) return;
+          instances.Add(qap);
+        }
+      });
+      return instances;
+    }
+
+    private static List<QuadraticAssignmentProblem> Get20DifferentClasses() {
       var sync = new object();
 
@@ -36,52 +87,62 @@
         });
       }
-
-      /*var classes = instances.Select(InstanceDescriptor.FromProblemOnly).GroupBy(x => x.Cls);
-      foreach (var cls in classes.OrderBy(x => x.Key)) {
-        Console.WriteLine("{0};{1}", cls.Key, cls.Count());
-      }*/
-
-      //var kb = GenerateKnowledgeBase(instances, 200, localOptima: false);
-
-
-      //var paths = new[] { 1, 2, 3, 5, 10, 20, 30, 50, 100, 200 };
-      //var kbExplorers = paths.Select(x => new PathRelinkingExplorer() { Paths = x }).ToArray();
-      //var expExplorers = paths.Select(x => new PathRelinkingExplorer() { Paths = x }).ToArray();
-      //ExploreSelfIdentification(instances, kbExplorers, expExplorers);
-
-      var iterations = new[] { 100, 1000, 10000, 100000 };
-      var kbExplorers = iterations.Select(x => new RandomWalkExplorer() { Iterations = x }).ToArray();
-      var expExplorers = iterations.Select(x => new RandomWalkExplorer() { Iterations = x }).ToArray();
-      ExploreSelfIdentification(instances, kbExplorers, expExplorers);
-
-      //ExploreMemPrIdentification(instances);
-    }
-
-    private static List<InstanceDescriptor> GenerateKnowledgeBase(List<QuadraticAssignmentProblem> instances, InstanceExplorer explorer) {
-      var headerPrinted = false;
-      var sync = new object();
-      var kb = new List<InstanceDescriptor>();
-      Parallel.ForEach(instances.OrderBy(x => x.Weights.Rows), qap => {
+      return instances;
+    }
+
+    private static List<InstanceDescriptor> GenerateData(List<QuadraticAssignmentProblem> instances, InstanceExplorer explorer, bool parallel = false) {
+      var sync = new object();
+      var data = new List<InstanceDescriptor>();
+      Action<QuadraticAssignmentProblem> body = (qap) => {
         var instance = explorer.Explore(qap);
-        lock (sync) {
-          if (!headerPrinted) {
-            headerPrinted = true;
-            Console.WriteLine(string.Join(";",
-              new [] { "Name", "Cls", "Dimension" }
-              .Concat(instance.FeatureNames)));
-          }
-          PrintInstanceLine(instance);
-          kb.Add(instance);
-        }
-      });
-      var standardizer = InstancesStandardizer.Create(kb);
-      var normalizedKb = kb.Select(x => new InstanceDescriptor(x)).ToList();
-      standardizer.Apply(normalizedKb);
-      Console.WriteLine();
-      foreach (var instance in kb) {
-        PrintInstanceLine(instance);
-      }
-      //return normalizedKb;
-      return kb;
+        if (instance == null) return;
+        lock (sync) {
+          data.Add(instance);
+        }
+      };
+      if (parallel) {
+        Parallel.ForEach(instances, body);
+      } else {
+        foreach (var qap in instances) body(qap);
+      }
+      return data;
+    }
+
+    private static MatchResult Compare(List<InstanceDescriptor> training, List<InstanceDescriptor> test) {
+      int exactCount = 0, clsCount = 0, totalCount = 0;
+      int exactRank = 0, clsRank = 0;
+      foreach (var e in test) {
+        var ordered = training.OrderBy(x => x.CalculateSimilarity(e)).ToList();
+        var bestMatch = ordered.First();
+        if (bestMatch.Cls == e.Cls) clsCount++;
+        if (bestMatch.Name == e.Name) exactCount++;
+        totalCount++;
+        clsRank += ordered.FindIndex((id) => id.Cls == e.Cls) + 1;
+        exactRank += ordered.FindIndex((id) => id.Name == e.Name) + 1;
+      }
+
+      return new MatchResult() {
+        ExactCount = exactCount,
+        ClsCount = clsCount,
+        TotalCount = totalCount,
+        ExactAverageRank = exactRank / (double)totalCount,
+        ClsAverageRank = clsRank / (double)totalCount
+      };
+    }
+
+    private static void PrintMatchResult(MatchResult result) {
+      Console.WriteLine("{0}\t{1}\t{2}\t{3:F2}\t{4:F2}",
+        result.ExactCount, result.ClsCount, result.TotalCount,
+        result.ExactAverageRank, result.ClsAverageRank);
+    }
+
+    private static void PrintData(List<InstanceDescriptor> instances) {
+      using (var iter = instances.GetEnumerator()) {
+        if (!iter.MoveNext()) return;
+        Console.WriteLine(string.Join(";", new[] {"Name", "Cls", "Dimension"}
+          .Concat(iter.Current != null ? iter.Current.FeatureNames : new [] { "(null)" })));
+        do {
+          PrintInstanceLine(iter.Current);
+        } while (iter.MoveNext());
+      }
     }
 
@@ -92,101 +153,72 @@
     }
 
-    private static void ExploreSelfIdentification(List<QuadraticAssignmentProblem> instances, InstanceExplorer[] kbExplorer, InstanceExplorer[] expExporer) {
-      var sync = new object();
-      Console.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}\t{5}", "Repetition", "KB-Effort", "Exp-Effort",
-        "Exact-Hits", "No-Hits", "Exact-Rank");
-      var kbSeeds = Enumerable.Range(0, 20).ToList();
-      var expSeeds = Enumerable.Range(20, 20).ToList();
-      for (var r = 0; r < 10; r++) {
-        var rlokal = r;
-        Parallel.ForEach(kbExplorer, kbPaths => {
-        //foreach (var kbPaths in kbExplorer) {
-          foreach (var expPaths in expExporer) {
-            //if (expPaths > kbPaths) continue;
-            var kb = new List<InstanceDescriptor>();
-            var exp = new List<InstanceDescriptor>();
-            foreach (var qap in instances) {
-              kb.Add(kbPaths.Explore(qap, kbSeeds[rlokal]));
-              exp.Add(expPaths.Explore(qap, expSeeds[rlokal]));
-            }
-            var standardizer = InstancesStandardizer.Create(kb);
-            standardizer.Apply(kb);
-            standardizer.Apply(exp);
-            int exactCount = 0, clsCount = 0, missedCount = 0;
-            int exactRank = 0, clsRank = 0;
-            foreach (var e in exp) {
-              var ordered = kb.OrderBy(x => x.CalculateSimilarity(e)).ToList();
-              var bestMatch = ordered.First();
-              if (bestMatch.Cls == e.Cls) {
-                clsCount++;
-                if (bestMatch.Name == e.Name) exactCount++;
-              }
-              else missedCount++;
-              clsRank += ordered.FindIndex((id) => id.Cls == e.Cls) + 1;
-              exactRank += ordered.FindIndex((id) => id.Name == e.Name) + 1;
-            }
-            lock (sync) {
-              Console.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}\t{5:F2}", rlokal, kbPaths.Effort, expPaths.Effort, exactCount,
-                missedCount, exactRank / (double) exp.Count);
-            }
+    private static void ExploreMatching(List<QuadraticAssignmentProblem> instances, InstanceExplorer[] trainingExplorers, InstanceExplorer[] testExporers, bool parallel = false) {
+      var sync = new object();
+      var rand = new Random();
+      var first = rand.Next();
+      var second = rand.Next();
+      while (first == second) second = rand.Next();
+
+      var knowledgeBase = new Dictionary<InstanceExplorer, Tuple<InstancesStandardizer, List<InstanceDescriptor>>>();
+      Action<InstanceExplorer> trainingBody = (kbExplorer) => {
+        var trainingData = GenerateData(instances, kbExplorer, parallel);
+        var standardizer = InstancesStandardizer.Create(trainingData);
+        standardizer.Apply(trainingData);
+        lock (sync) {
+          knowledgeBase.Add(kbExplorer, Tuple.Create(standardizer, trainingData));
+        }
+      };
+      if (parallel) {
+        Parallel.ForEach(trainingExplorers, trainingBody);
+      } else {
+        foreach (var kbExplorer in trainingExplorers) trainingBody(kbExplorer);
+      }
+
+      var experimentBase = new Dictionary<InstanceExplorer, List<InstanceDescriptor>>();
+      Action<InstanceExplorer> testBody = (expExplorer) => {
+        var trainingData = GenerateData(instances, expExplorer, parallel);
+        lock (sync) {
+          experimentBase.Add(expExplorer, trainingData);
+        }
+      };
+      if (parallel) {
+        Parallel.ForEach(testExporers, testBody);
+      } else {
+        foreach (var expExplorer in testExporers) testBody(expExplorer);
+      }
+
+      var data = from kb in knowledgeBase
+        from exp in experimentBase
+        select new { Training = kb, Test = exp };
+
+      if (parallel) {
+        Parallel.ForEach(data, (point) => {
+          var normalizedTest = point.Test.Value.Select(x => new InstanceDescriptor(x)).ToList();
+          point.Training.Value.Item1.Apply(normalizedTest);
+          var result = Compare(point.Training.Value.Item2, normalizedTest);
+          lock (sync) {
+            Console.WriteLine("{0}\t{1}\t{2}\t{3:F2}\t{4}\t{5:F2}\t{6}",
+              point.Training.Key.Effort, point.Test.Key.Effort, result.ExactCount,
+              result.ExactAverageRank, result.ClsCount, result.ClsAverageRank, result.TotalCount);
           }
         });
-        //}
-      }
-    }
-    
-    private static void ExploreMemPrIdentification(List<QuadraticAssignmentProblem> instances) {
-      var sync = new object();
-      Console.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}\t{5}", "Repetition", "KB-Runtime", "Exp-Runtime",
-        "Exact-Hits", "No-Hits", "Average-Rank");
-      var paths = new[] { 10 };
-      var repetitions = 10;
-      var kbSeeds = Enumerable.Range(0, repetitions).ToList();
-      var expSeeds = Enumerable.Range(repetitions, repetitions).ToList();
-      for (var r = 0; r < repetitions; r++) {
-        var rlokal = r;
-        Parallel.ForEach(paths, kbPaths => {
-          var memPr = new PermutationMemPR();
-          foreach (var expPaths in paths) {
-            //if (expPaths > kbPaths) continue;
-            var kb = new List<InstanceDescriptor>();
-            var exp = new List<InstanceDescriptor>();
-            foreach (var qap in instances.Select(x => (QuadraticAssignmentProblem)x.Clone())) {
-              memPr.Problem = qap;
-              memPr.Prepare(true);
-              memPr.Seed = kbSeeds[rlokal];
-              memPr.MaximumExecutionTime = TimeSpan.FromSeconds(kbPaths);
-              memPr.StartSync();
-              if (memPr.Context.RelinkedPaths.IsEmpty) continue;
-              kb.Add(InstanceDescriptor.FromPaths(qap, memPr.Context.RelinkedPaths.Paths.ToList()));
-              memPr.Prepare(true);
-              memPr.Seed = expSeeds[rlokal];
-              memPr.MaximumExecutionTime = TimeSpan.FromSeconds(expPaths);
-              memPr.StartSync();
-              if (memPr.Context.RelinkedPaths.IsEmpty) continue;
-              exp.Add(InstanceDescriptor.FromPaths(qap, memPr.Context.RelinkedPaths.Paths.ToList()));
-            }
-            var standardizer = InstancesStandardizer.Create(kb);
-            standardizer.Apply(kb);
-            standardizer.Apply(exp);
-            int exactCount = 0, clsCount = 0, missedCount = 0;
-            int exactRank = 0, clsRank = 0;
-            foreach (var e in exp) {
-              var ordered = kb.OrderBy(x => x.CalculateSimilarity(e)).ToList();
-              var bestMatch = ordered.First();
-              if (bestMatch.Cls == e.Cls) {
-                clsCount++;
-                if (bestMatch.Name == e.Name) exactCount++;
-              } else missedCount++;
-              clsRank += ordered.FindIndex((id) => id.Cls == e.Cls) + 1;
-              exactRank += ordered.FindIndex((id) => id.Name == e.Name) + 1;
-            }
-            lock (sync) {
-              Console.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}\t{5:F2}", rlokal, kbPaths, expPaths, exactCount,
-                missedCount, exactRank / (double)exp.Count);
-            }
-          }
-        });
-      }
+      } else {
+        foreach (var point in data) {
+          var normalizedTest = point.Test.Value.Select(x => new InstanceDescriptor(x)).ToList();
+          point.Training.Value.Item1.Apply(normalizedTest);
+          var result = Compare(point.Training.Value.Item2, normalizedTest);
+          Console.WriteLine("{0}\t{1}\t{2}\t{3:F2}\t{4}\t{5:F2}\t{6}",
+            point.Training.Key.Effort, point.Test.Key.Effort, result.ExactCount,
+            result.ExactAverageRank, result.ClsCount, result.ClsAverageRank, result.TotalCount);
+        }
+      }
+    }
+
+    private class MatchResult {
+      public int ExactCount { get; set; }
+      public int ClsCount { get; set; }
+      public int TotalCount { get; set; }
+      public double ExactAverageRank { get; set; }
+      public double ClsAverageRank { get; set; }
     }
   }
