Changeset 14691 for branches/PerformanceComparison/ProblemInstanceIdentifier/Program.cs

Timestamp:

02/23/17 10:42:58 (8 years ago)

Author:

abeham

Message:

#2457: working on identification of problem instances

File:

• branches/PerformanceComparison/ProblemInstanceIdentifier/Program.cs (modified) (4 diffs)

branches/PerformanceComparison/ProblemInstanceIdentifier/Program.cs

@@ -9 +9 @@
 using HeuristicLab.Problems.Instances.QAPLIB;
 using HeuristicLab.Problems.QuadraticAssignment;
+using HeuristicLab.Random;
 
 namespace ProblemInstanceIdentifier {
@@ -18 +19 @@
     };
     static void Main(string[] args) {
+      var instances = Get20DifferentClasses();
+      //var instances = GetSomeRandomInstances(50);
+
+      /*var classes = instances.Select(InstanceDescriptor.FromProblemOnly).GroupBy(x => x.Cls);
+      foreach (var cls in classes.OrderBy(x => x.Key)) {
+        Console.WriteLine("{0};{1}", cls.Key, cls.Count());
+      }*/
+
+      var prExplorer = new PathRelinkingExplorer() {
+        LocalOptima = false,
+        Paths = 200
+      };
+      var prLocalExplorer = new PathRelinkingExplorer() {
+        LocalOptima = true,
+        Paths = 200
+      };
+      var memPrExplorer = new MemPRExplorer() {
+        IncludeLocalSearch = false,
+        Seconds = 10
+      };
+
+      var training = GenerateData(instances, prExplorer, parallel:true);
+      var standardizer = InstancesStandardizer.CreateAndApply(training);
+      var test = GenerateData(instances, prExplorer, parallel: false);
+      standardizer.Apply(test);
+      PrintMatchResult(Compare(training, test));
+
+      ExploreMatching(instances, new [] { prLocalExplorer }, new [] { memPrExplorer });
+    }
+
+    private static List<QuadraticAssignmentProblem> GetSomeRandomInstances(int totalInstances) {
+      var sync = new object();
+      var provider = new OneSizeInstanceProvider();
+      var instances = new List<QuadraticAssignmentProblem>();
+      var random = new FastRandom(0);
+      Parallel.ForEach(provider.GetDataDescriptors().Shuffle(random), desc => {
+        var qapData = provider.LoadData(desc);
+        if (qapData.Dimension < 25) return;
+        if (instances.Count >= totalInstances) return;
+        var qap = new QuadraticAssignmentProblem();
+        qap.Load(qapData);
+        lock (sync) {
+          if (instances.Count >= totalInstances) return;
+          instances.Add(qap);
+        }
+      });
+      return instances;
+    }
+
+    private static List<QuadraticAssignmentProblem> Get20DifferentClasses() {
       var sync = new object();
 
@@ -36 +87 @@
         });
       }
-
-      /*var classes = instances.Select(InstanceDescriptor.FromProblemOnly).GroupBy(x => x.Cls);
-      foreach (var cls in classes.OrderBy(x => x.Key)) {
-        Console.WriteLine("{0};{1}", cls.Key, cls.Count());
-      }*/
-
-      //var kb = GenerateKnowledgeBase(instances, 200, localOptima: false);
-
-
-      //var paths = new[] { 1, 2, 3, 5, 10, 20, 30, 50, 100, 200 };
-      //var kbExplorers = paths.Select(x => new PathRelinkingExplorer() { Paths = x }).ToArray();
-      //var expExplorers = paths.Select(x => new PathRelinkingExplorer() { Paths = x }).ToArray();
-      //ExploreSelfIdentification(instances, kbExplorers, expExplorers);
-
-      var iterations = new[] { 100, 1000, 10000, 100000 };
-      var kbExplorers = iterations.Select(x => new RandomWalkExplorer() { Iterations = x }).ToArray();
-      var expExplorers = iterations.Select(x => new RandomWalkExplorer() { Iterations = x }).ToArray();
-      ExploreSelfIdentification(instances, kbExplorers, expExplorers);
-
-      //ExploreMemPrIdentification(instances);
-    }
-
-    private static List<InstanceDescriptor> GenerateKnowledgeBase(List<QuadraticAssignmentProblem> instances, InstanceExplorer explorer) {
-      var headerPrinted = false;
-      var sync = new object();
-      var kb = new List<InstanceDescriptor>();
-      Parallel.ForEach(instances.OrderBy(x => x.Weights.Rows), qap => {
+      return instances;
+    }
+
+    private static List<InstanceDescriptor> GenerateData(List<QuadraticAssignmentProblem> instances, InstanceExplorer explorer, bool parallel = false) {
+      var sync = new object();
+      var data = new List<InstanceDescriptor>();
+      Action<QuadraticAssignmentProblem> body = (qap) => {
         var instance = explorer.Explore(qap);
-        lock (sync) {
-          if (!headerPrinted) {
-            headerPrinted = true;
-            Console.WriteLine(string.Join(";",
-              new [] { "Name", "Cls", "Dimension" }
-              .Concat(instance.FeatureNames)));
-          }
-          PrintInstanceLine(instance);
-          kb.Add(instance);
-        }
-      });
-      var standardizer = InstancesStandardizer.Create(kb);
-      var normalizedKb = kb.Select(x => new InstanceDescriptor(x)).ToList();
-      standardizer.Apply(normalizedKb);
-      Console.WriteLine();
-      foreach (var instance in kb) {
-        PrintInstanceLine(instance);
-      }
-      //return normalizedKb;
-      return kb;
+        if (instance == null) return;
+        lock (sync) {
+          data.Add(instance);
+        }
+      };
+      if (parallel) {
+        Parallel.ForEach(instances, body);
+      } else {
+        foreach (var qap in instances) body(qap);
+      }
+      return data;
+    }
+
+    private static MatchResult Compare(List<InstanceDescriptor> training, List<InstanceDescriptor> test) {
+      int exactCount = 0, clsCount = 0, totalCount = 0;
+      int exactRank = 0, clsRank = 0;
+      foreach (var e in test) {
+        var ordered = training.OrderBy(x => x.CalculateSimilarity(e)).ToList();
+        var bestMatch = ordered.First();
+        if (bestMatch.Cls == e.Cls) clsCount++;
+        if (bestMatch.Name == e.Name) exactCount++;
+        totalCount++;
+        clsRank += ordered.FindIndex((id) => id.Cls == e.Cls) + 1;
+        exactRank += ordered.FindIndex((id) => id.Name == e.Name) + 1;
+      }
+
+      return new MatchResult() {
+        ExactCount = exactCount,
+        ClsCount = clsCount,
+        TotalCount = totalCount,
+        ExactAverageRank = exactRank / (double)totalCount,
+        ClsAverageRank = clsRank / (double)totalCount
+      };
+    }
+
+    private static void PrintMatchResult(MatchResult result) {
+      Console.WriteLine("{0}\t{1}\t{2}\t{3:F2}\t{4:F2}",
+        result.ExactCount, result.ClsCount, result.TotalCount,
+        result.ExactAverageRank, result.ClsAverageRank);
+    }
+
+    private static void PrintData(List<InstanceDescriptor> instances) {
+      using (var iter = instances.GetEnumerator()) {
+        if (!iter.MoveNext()) return;
+        Console.WriteLine(string.Join(";", new[] {"Name", "Cls", "Dimension"}
+          .Concat(iter.Current != null ? iter.Current.FeatureNames : new [] { "(null)" })));
+        do {
+          PrintInstanceLine(iter.Current);
+        } while (iter.MoveNext());
+      }
     }
 
@@ -92 +153 @@
     }
 
-    private static void ExploreSelfIdentification(List<QuadraticAssignmentProblem> instances, InstanceExplorer[] kbExplorer, InstanceExplorer[] expExporer) {
-      var sync = new object();
-      Console.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}\t{5}", "Repetition", "KB-Effort", "Exp-Effort",
-        "Exact-Hits", "No-Hits", "Exact-Rank");
-      var kbSeeds = Enumerable.Range(0, 20).ToList();
-      var expSeeds = Enumerable.Range(20, 20).ToList();
-      for (var r = 0; r < 10; r++) {
-        var rlokal = r;
-        Parallel.ForEach(kbExplorer, kbPaths => {
-        //foreach (var kbPaths in kbExplorer) {
-          foreach (var expPaths in expExporer) {
-            //if (expPaths > kbPaths) continue;
-            var kb = new List<InstanceDescriptor>();
-            var exp = new List<InstanceDescriptor>();
-            foreach (var qap in instances) {
-              kb.Add(kbPaths.Explore(qap, kbSeeds[rlokal]));
-              exp.Add(expPaths.Explore(qap, expSeeds[rlokal]));
-            }
-            var standardizer = InstancesStandardizer.Create(kb);
-            standardizer.Apply(kb);
-            standardizer.Apply(exp);
-            int exactCount = 0, clsCount = 0, missedCount = 0;
-            int exactRank = 0, clsRank = 0;
-            foreach (var e in exp) {
-              var ordered = kb.OrderBy(x => x.CalculateSimilarity(e)).ToList();
-              var bestMatch = ordered.First();
-              if (bestMatch.Cls == e.Cls) {
-                clsCount++;
-                if (bestMatch.Name == e.Name) exactCount++;
-              }
-              else missedCount++;
-              clsRank += ordered.FindIndex((id) => id.Cls == e.Cls) + 1;
-              exactRank += ordered.FindIndex((id) => id.Name == e.Name) + 1;
-            }
-            lock (sync) {
-              Console.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}\t{5:F2}", rlokal, kbPaths.Effort, expPaths.Effort, exactCount,
-                missedCount, exactRank / (double) exp.Count);
-            }
+    private static void ExploreMatching(List<QuadraticAssignmentProblem> instances, InstanceExplorer[] trainingExplorers, InstanceExplorer[] testExporers, bool parallel = false) {
+      var sync = new object();
+      var rand = new Random();
+      var first = rand.Next();
+      var second = rand.Next();
+      while (first == second) second = rand.Next();
+
+      var knowledgeBase = new Dictionary<InstanceExplorer, Tuple<InstancesStandardizer, List<InstanceDescriptor>>>();
+      Action<InstanceExplorer> trainingBody = (kbExplorer) => {
+        var trainingData = GenerateData(instances, kbExplorer, parallel);
+        var standardizer = InstancesStandardizer.Create(trainingData);
+        standardizer.Apply(trainingData);
+        lock (sync) {
+          knowledgeBase.Add(kbExplorer, Tuple.Create(standardizer, trainingData));
+        }
+      };
+      if (parallel) {
+        Parallel.ForEach(trainingExplorers, trainingBody);
+      } else {
+        foreach (var kbExplorer in trainingExplorers) trainingBody(kbExplorer);
+      }
+
+      var experimentBase = new Dictionary<InstanceExplorer, List<InstanceDescriptor>>();
+      Action<InstanceExplorer> testBody = (expExplorer) => {
+        var trainingData = GenerateData(instances, expExplorer, parallel);
+        lock (sync) {
+          experimentBase.Add(expExplorer, trainingData);
+        }
+      };
+      if (parallel) {
+        Parallel.ForEach(testExporers, testBody);
+      } else {
+        foreach (var expExplorer in testExporers) testBody(expExplorer);
+      }
+
+      var data = from kb in knowledgeBase
+        from exp in experimentBase
+        select new { Training = kb, Test = exp };
+
+      if (parallel) {
+        Parallel.ForEach(data, (point) => {
+          var normalizedTest = point.Test.Value.Select(x => new InstanceDescriptor(x)).ToList();
+          point.Training.Value.Item1.Apply(normalizedTest);
+          var result = Compare(point.Training.Value.Item2, normalizedTest);
+          lock (sync) {
+            Console.WriteLine("{0}\t{1}\t{2}\t{3:F2}\t{4}\t{5:F2}\t{6}",
+              point.Training.Key.Effort, point.Test.Key.Effort, result.ExactCount,
+              result.ExactAverageRank, result.ClsCount, result.ClsAverageRank, result.TotalCount);
           }
         });
-        //}
-      }
-    }
-    
-    private static void ExploreMemPrIdentification(List<QuadraticAssignmentProblem> instances) {
-      var sync = new object();
-      Console.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}\t{5}", "Repetition", "KB-Runtime", "Exp-Runtime",
-        "Exact-Hits", "No-Hits", "Average-Rank");
-      var paths = new[] { 10 };
-      var repetitions = 10;
-      var kbSeeds = Enumerable.Range(0, repetitions).ToList();
-      var expSeeds = Enumerable.Range(repetitions, repetitions).ToList();
-      for (var r = 0; r < repetitions; r++) {
-        var rlokal = r;
-        Parallel.ForEach(paths, kbPaths => {
-          var memPr = new PermutationMemPR();
-          foreach (var expPaths in paths) {
-            //if (expPaths > kbPaths) continue;
-            var kb = new List<InstanceDescriptor>();
-            var exp = new List<InstanceDescriptor>();
-            foreach (var qap in instances.Select(x => (QuadraticAssignmentProblem)x.Clone())) {
-              memPr.Problem = qap;
-              memPr.Prepare(true);
-              memPr.Seed = kbSeeds[rlokal];
-              memPr.MaximumExecutionTime = TimeSpan.FromSeconds(kbPaths);
-              memPr.StartSync();
-              if (memPr.Context.RelinkedPaths.IsEmpty) continue;
-              kb.Add(InstanceDescriptor.FromPaths(qap, memPr.Context.RelinkedPaths.Paths.ToList()));
-              memPr.Prepare(true);
-              memPr.Seed = expSeeds[rlokal];
-              memPr.MaximumExecutionTime = TimeSpan.FromSeconds(expPaths);
-              memPr.StartSync();
-              if (memPr.Context.RelinkedPaths.IsEmpty) continue;
-              exp.Add(InstanceDescriptor.FromPaths(qap, memPr.Context.RelinkedPaths.Paths.ToList()));
-            }
-            var standardizer = InstancesStandardizer.Create(kb);
-            standardizer.Apply(kb);
-            standardizer.Apply(exp);
-            int exactCount = 0, clsCount = 0, missedCount = 0;
-            int exactRank = 0, clsRank = 0;
-            foreach (var e in exp) {
-              var ordered = kb.OrderBy(x => x.CalculateSimilarity(e)).ToList();
-              var bestMatch = ordered.First();
-              if (bestMatch.Cls == e.Cls) {
-                clsCount++;
-                if (bestMatch.Name == e.Name) exactCount++;
-              } else missedCount++;
-              clsRank += ordered.FindIndex((id) => id.Cls == e.Cls) + 1;
-              exactRank += ordered.FindIndex((id) => id.Name == e.Name) + 1;
-            }
-            lock (sync) {
-              Console.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}\t{5:F2}", rlokal, kbPaths, expPaths, exactCount,
-                missedCount, exactRank / (double)exp.Count);
-            }
-          }
-        });
-      }
+      } else {
+        foreach (var point in data) {
+          var normalizedTest = point.Test.Value.Select(x => new InstanceDescriptor(x)).ToList();
+          point.Training.Value.Item1.Apply(normalizedTest);
+          var result = Compare(point.Training.Value.Item2, normalizedTest);
+          Console.WriteLine("{0}\t{1}\t{2}\t{3:F2}\t{4}\t{5:F2}\t{6}",
+            point.Training.Key.Effort, point.Test.Key.Effort, result.ExactCount,
+            result.ExactAverageRank, result.ClsCount, result.ClsAverageRank, result.TotalCount);
+        }
+      }
+    }
+
+    private class MatchResult {
+      public int ExactCount { get; set; }
+      public int ClsCount { get; set; }
+      public int TotalCount { get; set; }
+      public double ExactAverageRank { get; set; }
+      public double ClsAverageRank { get; set; }
     }
   }