Changeset 16096 for branches/2457_ExpertSystem/ProblemInstanceIdentifier

Timestamp:

08/29/18 18:16:05 (6 years ago)

Author:

abeham

Message:

#2457:

• Changed calculation of correlation length (using limit introduced Hordijk 1996)
• Changed RuggednessCalculator (no more a HL item)
• Added additional, information-analysis-based features for directed walks
• Added generic DirectedWalk algorithm (as described in thesis)
• Made OneSizeInstanceProvider parametrizable
• Adapted program for analyzing problem instance reidentification

Location:

branches/2457_ExpertSystem/ProblemInstanceIdentifier

Files:

• InstanceDescriptor.cs (modified) (4 diffs)
• InstanceExplorer.cs (modified) (10 diffs)
• ProblemInstanceIdentifier.csproj (modified) (1 diff)
• Program.cs (modified) (11 diffs)

branches/2457_ExpertSystem/ProblemInstanceIdentifier/InstanceDescriptor.cs

@@ -1 @@
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.Linq;
 using System.Text.RegularExpressions;
-using HeuristicLab.Analysis.FitnessLandscape;
 using HeuristicLab.Common;
-using HeuristicLab.Encodings.PermutationEncoding;
 using HeuristicLab.Problems.QuadraticAssignment;
 
@@ -16 +13 @@
     public string[] FeatureNames { get; set; }
     public double[] FeatureValues { get; set; }
+
+    public double DescriptionEffort { get; set; }
     
     private InstanceDescriptor() { }
@@ -33 +32 @@
       FeatureNames = (string[])other.FeatureNames.Clone();
       FeatureValues = (double[]) other.FeatureValues.Clone();
+      DescriptionEffort = other.DescriptionEffort;
     }
 
@@ -41 +41 @@
         Dimension = qap.Weights.Rows,
         FeatureNames = new string[0],
-        FeatureValues = new double[0]
-      };
-    }
-
-    public static InstanceDescriptor FromPaths(QuadraticAssignmentProblem qap, List<List<Tuple<Permutation, double>>> trajectories) {
-      var result = PermutationPathAnalysis.GetCharacteristics(trajectories);
-
-      return new InstanceDescriptor() {
-        Name = qap.Name,
-        Cls = GetClass(qap.Name),
-        Dimension = qap.Weights.Rows,
-        FeatureNames = result.GetNames(),
-        FeatureValues = result.GetValues()
+        FeatureValues = new double[0],
+        DescriptionEffort = 0
       };
     }

branches/2457_ExpertSystem/ProblemInstanceIdentifier/InstanceExplorer.cs

@@ -1 +1 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
-using HeuristicLab.Algorithms.MemPR.Permutation;
 using HeuristicLab.Analysis.FitnessLandscape;
 using HeuristicLab.Data;
 using HeuristicLab.Encodings.PermutationEncoding;
 using HeuristicLab.Problems.QuadraticAssignment;
-using HeuristicLab.Random;
 using HeuristicLab.SequentialEngine;
 
@@ -21 +20 @@
   public class PathRelinkingExplorer : InstanceExplorer {
     public int Paths { get; set; }
-    public bool LocalOptima { get; set; }
+    public QAPDirectedWalk.WalkType Type { get; set; }
 
     public override string Name { get { return "Path-Relinking Explorer"; } }
     public override int Effort { get { return Paths; } }
 
+    public ISet<string> Features { get; set; }
+
     public PathRelinkingExplorer() {
       
@@ -31 +32 @@
 
     public override InstanceDescriptor Explore(QuadraticAssignmentProblem problem, int? seed = null) {
-      var walk = new QAPDirectedWalk {
+      var calculator = new QAPDirectedWalk {
         Problem = problem,
         BestImprovement = true,
         Paths = Paths,
         Seed = seed,
-        LocalOptima = LocalOptima
-      };
-      var features = walk.Calculate().ToDictionary(x => x.Name, x => x.Value);
-
-      return new InstanceDescriptor(problem.Name, InstanceDescriptor.GetClass(problem.Name), problem.Weights.Rows,
-        features.Keys.ToArray(), features.Values.Select(x => ((DoubleValue)x).Value).ToArray());
-    }
-  }
-
-  public class PathRelinkingOldFeaturedExplorer : InstanceExplorer {
-    public int Paths { get; set; }
-    public bool LocalOptima { get; set; }
-
-    public override string Name { get { return "Path-Relinking Explorer"; } }
-    public override int Effort { get { return Paths; } }
-
-    public PathRelinkingOldFeaturedExplorer() {
-
-    }
-
-    public override InstanceDescriptor Explore(QuadraticAssignmentProblem problem, int? seed = null) {
-      var random = new MersenneTwister();
-      var samples = QAPDirectedWalk.CalculateRelinkingPoints(random, problem, Paths, LocalOptima);
-      var trajectories = QAPDirectedWalk.Run(random, problem, samples);
-
-      double avgAc1 = 0, avgCorLen = 0, avgIc = 0, avgDbi = 0, avgPic = 0, avgIs = 0, avgDiv = 0,
-        avgReg = 0, avgEnt = 0, avgPkIc = 0, avgPkDbi = 0;
-      int count = 0;
-      foreach (var t in trajectories) {
-        var trail = t.Select(x => x.Item2).ToArray();
-        if (trail.Length < 4) continue;
-        count++;
-        double[] acf;
-        var len = RuggednessCalculator.CalculateCorrelationLength(trail, out acf);
-        avgAc1 += acf[0];
-        avgCorLen += len;
-        var analysis = new InformationAnalysis(trail, 20, 2);
-        avgIc += analysis.InformationContent[0];
-        avgDbi += analysis.DensityBasinInformation[0];
-        avgPic += analysis.PartialInformationContent[0];
-        avgIs += analysis.InformationStability;
-        avgDiv += analysis.Diversity;
-        avgReg += analysis.Regularity;
-        avgEnt += analysis.TotalEntropy[0];
-        avgPkIc += analysis.PeakInformationContent.Value;
-        avgPkDbi += analysis.PeakDensityBasinInformation.Value;
-      }
-      avgAc1 /= count;
-      avgCorLen /= count;
-      avgIc /= count;
-      avgDbi /= count;
-      avgPic /= count;
-      avgIs /= count;
-      avgDiv /= count;
-      avgReg /= count;
-      avgEnt /= count;
-      avgPkIc /= count;
-      avgPkDbi /= count;
-
-      return new InstanceDescriptor(problem.Name, InstanceDescriptor.GetClass(problem.Name), problem.Weights.Rows,
-        new[] { "Autocorrelation(1)", "CorrelationLength", "InformationContent", "DensityBasinInformation",
-          "PartialInformationContent", "InformationStability", "Diversity", "Regularity", "TotalEntropy",
-          "PeakInformationContent", "PeakDensityBasinInformation" },
-        new[] { avgAc1, avgCorLen, avgIc, avgDbi,
-          avgPic, avgIs, avgDiv, avgReg, avgEnt,
-          avgPkIc, avgPkDbi });
-    }
-  }
-
+        Type = Type
+      };
+      if (Features != null && Features.Count > 0) {
+        foreach (var c in calculator.Characteristics.ToList()) {
+          calculator.Characteristics.SetItemCheckedState(c, Features.Contains(c.Value));
+        }
+      }
+      var result = calculator.Calculate().ToDictionary(x => x.Name, x => x.Value);
+      var evaluations = ((IntValue)result["EvaluatedSolutions"]).Value;
+      result.Remove("EvaluatedSolutions");
+      if (Features != null && result.Count != Features.Count) throw new InvalidOperationException("Not all features in results");
+
+      return new InstanceDescriptor(problem.Name, InstanceDescriptor.GetClass(problem.Name), problem.Weights.Rows,
+        result.Keys.ToArray(), result.Values.Select(x => ((DoubleValue)x).Value).ToArray()) { DescriptionEffort = evaluations };
+    }
+  }
 
   public class RandomWalkExplorer : InstanceExplorer {
@@ -111 +59 @@
     public override string Name { get { return "Random-Walk Explorer"; } }
     public override int Effort { get { return Iterations; } }
+
+    public ISet<string> Features { get; set; }
 
     public RandomWalkExplorer() {
@@ -127 +77 @@
       walk.MutatorParameter.Value = walk.MutatorParameter.ValidValues.First(x => x is Swap2Manipulator);
       var calculator = new RandomWalkCalculator(walk) { Problem = problem };
-      var features = calculator.Calculate().ToDictionary(x => x.Name, x => x.Value);
-      
-      return new InstanceDescriptor(problem.Name, InstanceDescriptor.GetClass(problem.Name), problem.Weights.Rows,
-        features.Keys.ToArray(), features.Values.Select(x => ((DoubleValue)x).Value).ToArray());
+      foreach (var c in calculator.Characteristics.ToList()) {
+        calculator.Characteristics.SetItemCheckedState(c, Features.Contains(c.Value));
+      }
+      var result = calculator.Calculate().ToDictionary(x => x.Name, x => x.Value);
+      if (Features != null && result.Count != Features.Count) throw new InvalidOperationException("Not all features in results");
+
+      return new InstanceDescriptor(problem.Name, InstanceDescriptor.GetClass(problem.Name), problem.Weights.Rows,
+        result.Keys.ToArray(), result.Values.Select(x => ((DoubleValue)x).Value).ToArray());
     }
   }
@@ -139 +93 @@
 
     public override string Name { get { return "Adaptive-Walk Explorer"; } }
-    public override int Effort { get { return Iterations * SampleSize; } }
+    private int _effort;
+    public override int Effort { get { return _effort; } }
+
+    public ISet<string> Features { get; set; }
 
     public AdaptiveWalkExplorer() {
-
+      _effort = 0;
     }
 
@@ -157 +114 @@
       walk.MutatorParameter.Value = walk.MutatorParameter.ValidValues.First(x => x is Swap2Manipulator);
       var calculator = new AdaptiveWalkCalculator(walk) { Problem = problem };
-      var features = calculator.Calculate().ToDictionary(x => x.Name, x => x.Value);
-
-      return new InstanceDescriptor(problem.Name, InstanceDescriptor.GetClass(problem.Name), problem.Weights.Rows,
-        features.Keys.ToArray(), features.Values.Select(x => ((DoubleValue)x).Value).ToArray());
+      foreach (var c in calculator.Characteristics.ToList()) {
+        calculator.Characteristics.SetItemCheckedState(c, Features.Contains(c.Value));
+      }
+      var result = calculator.Calculate().ToDictionary(x => x.Name, x => x.Value);
+      if (Features != null && result.Count != Features.Count) throw new InvalidOperationException("Not all features in results");
+
+      return new InstanceDescriptor(problem.Name, InstanceDescriptor.GetClass(problem.Name), problem.Weights.Rows,
+        result.Keys.ToArray(), result.Values.Select(x => ((DoubleValue)x).Value).ToArray());
     }
   }
@@ -170 +131 @@
     public override string Name { get { return "Up/Down-Walk Explorer"; } }
     public override int Effort { get { return Iterations * SampleSize; } }
+
+    public ISet<string> Features { get; set; }
 
     public UpDownWalkExplorer() {
@@ -187 +150 @@
       walk.MutatorParameter.Value = walk.MutatorParameter.ValidValues.First(x => x is Swap2Manipulator);
       var calculator = new UpDownWalkCalculator(walk) {  Problem = problem };
-      var features = calculator.Calculate().ToDictionary(x => x.Name, x => x.Value);
-
-      return new InstanceDescriptor(problem.Name, InstanceDescriptor.GetClass(problem.Name), problem.Weights.Rows,
-        features.Keys.ToArray(), features.Values.Select(x => ((DoubleValue)x).Value).ToArray());
-    }
-  }
-
+      foreach (var c in calculator.Characteristics.ToList()) {
+        calculator.Characteristics.SetItemCheckedState(c, Features.Contains(c.Value));
+      }
+      var result = calculator.Calculate().ToDictionary(x => x.Name, x => x.Value);
+      if (Features != null && result.Count != Features.Count) throw new InvalidOperationException("Not all features in results");
+
+      return new InstanceDescriptor(problem.Name, InstanceDescriptor.GetClass(problem.Name), problem.Weights.Rows,
+        result.Keys.ToArray(), result.Values.Select(x => ((DoubleValue)x).Value).ToArray());
+    }
+  }
+  /*
   public class MemPRExplorer : InstanceExplorer {
     public int Seconds { get; set; }
@@ -230 +197 @@
       return new InstanceDescriptor(problem.Name, InstanceDescriptor.GetClass(problem.Name), problem.Weights.Rows, resultNames, result);
     }
-  }
+  }*/
 }

branches/2457_ExpertSystem/ProblemInstanceIdentifier/ProblemInstanceIdentifier.csproj

@@ -97 +97 @@
   </ItemGroup>
   <ItemGroup>
-    <ProjectReference Include="..\HeuristicLab.Algorithms.MemPR\3.3\HeuristicLab.Algorithms.MemPR-3.3.csproj">
-      <Project>{9d274421-6332-4fbc-aae4-467ace27c368}</Project>
-      <Name>HeuristicLab.Algorithms.MemPR-3.3</Name>
-    </ProjectReference>
     <ProjectReference Include="..\HeuristicLab.Analysis.FitnessLandscape\3.3\HeuristicLab.Analysis.FitnessLandscape-3.3.csproj">
       <Project>{5fbdcd4a-3c2a-4ec6-83ce-34b29f43621a}</Project>

branches/2457_ExpertSystem/ProblemInstanceIdentifier/Program.cs

@@ -2 +2 @@
 using System.Collections.Generic;
 using System.Globalization;
+using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
-using HeuristicLab.PluginInfrastructure;
-using HeuristicLab.Problems.Instances;
+using HeuristicLab.Analysis.FitnessLandscape;
 using HeuristicLab.Problems.Instances.QAPLIB;
 using HeuristicLab.Problems.QuadraticAssignment;
@@ -13 +13 @@
   class Program {
     static void Main(string[] args) {
-      //var instances = Get20DifferentClasses();
-      var instances = GetSomeRandomInstances(50, 13);
 
       /*var classes = instances.Select(InstanceDescriptor.FromProblemOnly).GroupBy(x => x.Cls);
@@ -21 +19 @@
       }*/
 
-      var prExplorer = new PathRelinkingExplorer() {
-        LocalOptima = false,
-        Paths = 200
-      };
-      var prOldExplorer = new PathRelinkingOldFeaturedExplorer() {
-        LocalOptima = false,
-        Paths = 200
-      };
-
-      var prLocalExplorer = new PathRelinkingExplorer() {
-        LocalOptima = true,
-        Paths = 200
-      };
-      var prOldLocalExplorer = new PathRelinkingOldFeaturedExplorer() {
-        LocalOptima = true,
-        Paths = 200
-      };
-
-      var memPrExplorer = new MemPRExplorer() {
-        IncludeLocalSearch = false,
-        Seconds = 10
-      };
-
-      //var training = GenerateData(instances, prOldExplorer, parallel: true);
+      var dwFeatureSets = new Dictionary<string, HashSet<string>>() {
+        { "SBF", new HashSet<string>() { "Sharpness", "Bumpiness", "Flatness" } },
+        { "FRQ", new HashSet<string>() { "DownQ2", "NeutQ2", "UpQ2", "DownQ1", "NeutQ1", "UpQ1", "DownQ3", "NeutQ3", "UpQ3" } },
+        { "SBF+FRQ", new HashSet<string>() { "Sharpness", "Bumpiness", "Flatness", "DownQ2", "NeutQ2", "UpQ2", "DownQ1", "NeutQ1", "UpQ1", "DownQ3", "NeutQ3", "UpQ3" } },
+        { "RUG", new HashSet<string>() { "AutoCorrelation1", "CorrelationLength" } },
+        { "IAL", new HashSet<string>() { "InformationContent", "DensityBasinInformation", "PartialInformationContent", "Regularity", "TotalEntropy", "PeakInformationContent", "PeakDensityBasinInformation" } },
+        { "RUG+IAL", new HashSet<string>() { "AutoCorrelation1", "CorrelationLength", "InformationContent", "DensityBasinInformation", "PartialInformationContent", "Regularity", "TotalEntropy", "PeakInformationContent", "PeakDensityBasinInformation" } },
+        { "SBF+IAL", new HashSet<string>() { "Sharpness", "Bumpiness", "Flatness", "InformationContent", "DensityBasinInformation", "PartialInformationContent", "Regularity", "TotalEntropy", "PeakInformationContent", "PeakDensityBasinInformation" } },
+        { "FRQ+IAL", new HashSet<string>() { "DownQ2", "NeutQ2", "UpQ2", "DownQ1", "NeutQ1", "UpQ1", "DownQ3", "NeutQ3", "UpQ3", "InformationContent", "DensityBasinInformation", "PartialInformationContent", "Regularity", "TotalEntropy", "PeakInformationContent", "PeakDensityBasinInformation" } },
+        { "SBF+FRQ+IAL", new HashSet<string>() { "Sharpness", "Bumpiness", "Flatness", "DownQ2", "NeutQ2", "UpQ2", "DownQ1", "NeutQ1", "UpQ1", "DownQ3", "NeutQ3", "UpQ3", "InformationContent", "DensityBasinInformation", "PartialInformationContent", "Regularity", "TotalEntropy", "PeakInformationContent", "PeakDensityBasinInformation" } },
+        { "SBF+FRQ+RUG+IAL", new HashSet<string>() { "Sharpness", "Bumpiness", "Flatness", "DownQ2", "NeutQ2", "UpQ2", "DownQ1", "NeutQ1", "UpQ1", "DownQ3", "NeutQ3", "UpQ3", "AutoCorrelation1", "CorrelationLength", "InformationContent", "DensityBasinInformation", "PartialInformationContent", "Regularity", "TotalEntropy", "PeakInformationContent", "PeakDensityBasinInformation" } },
+      };
+
+      var dwTypes = new Dictionary<string, QAPDirectedWalk.WalkType> {
+        { "(rr)-dw", QAPDirectedWalk.WalkType.RandomRandom },
+        { "(rl)-dw", QAPDirectedWalk.WalkType.RandomLocal },
+        { "(ll)-dw", QAPDirectedWalk.WalkType.LocalLocal },
+        { "(li)-dw", QAPDirectedWalk.WalkType.LocalInverse }
+      };
+
+      var dwPaths = new List<int> { 1, 2, 5, 10, 20, 50, 100, 200, 500 };
+
+      var random = new Random();
+      using (var writer = File.CreateText("results.csv")) {
+        writer.AutoFlush = true;
+        var header = string.Format("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\t{10}\t{11}",
+          "Rep", "FSet", "Type", "TrainEff", "TestEff", "ExCnt", "ExRnk", "ClsCnt", "ClsRnk", "TotCnt", "TrainEff2", "TestEff2");
+        writer.WriteLine(header);
+        Console.WriteLine(header);
+        foreach (var dim in new[] { 20, 30, 40, 50, 100 }) {
+          foreach (var feat in dwFeatureSets) {
+            foreach (var type in dwTypes) {
+              for (var rep = 0; rep < 10; rep++) {
+                var instances = GetSomeRandomInstances(dimension: dim, totalInstances: 20, seed: random.Next());
+
+                var explorers = from p in dwPaths
+                                select new PathRelinkingExplorer() { Features = feat.Value, Paths = p, Type = type.Value };
+                ExploreMatching(writer, instances, explorers.ToArray(), explorers.ToArray(), string.Format("{0}\t{1}\t{2}", rep, feat.Key, type.Key));
+              }
+            }
+          }
+        }
+      }
+
+
+      //var training = GenerateData(instances, rrDwExplorer, parallel: true);
       //var standardizer = InstancesStandardizer.CreateAndApply(training);
-      //var test = GenerateData(instances, prOldExplorer, parallel: true);
+      //var test = GenerateData(instances, rrDwExplorer, parallel: true);
       //standardizer.Apply(test);
       //PrintMatchResult(Compare(training, test));
@@ -65 +85 @@
       //new[] { 300, 1500, 3000, 6000, 15000, 30000, 60000 }.Select(x => new AdaptiveWalkExplorer() { Iterations = x / 10, SampleSize = 10 }).ToArray(),
       //parallel: true);
-      Console.WriteLine("=== Up/Down Walk ===");
-      ExploreMatching(instances,
-      new[] { 300, 1500, 3000, 6000, 15000, 30000, 60000 }.Select(x => new UpDownWalkExplorer() { Iterations = x / 10, SampleSize = 10 }).ToArray(),
-      new[] { 300, 1500, 3000, 6000, 15000, 30000, 60000 }.Select(x => new UpDownWalkExplorer() { Iterations = x / 10, SampleSize = 10 }).ToArray(),
-      parallel: true);
-    }
-
-    private static List<QuadraticAssignmentProblem> GetSomeRandomInstances(int totalInstances, int seed) {
+      //Console.WriteLine("=== Up/Down Walk ===");
+      //ExploreMatching(instances,
+      //new[] { 300, 1500, 3000, 6000, 15000, 30000, 60000 }.Select(x => new UpDownWalkExplorer() { Iterations = x / 10, SampleSize = 10 }).ToArray(),
+      //new[] { 300, 1500, 3000, 6000, 15000, 30000, 60000 }.Select(x => new UpDownWalkExplorer() { Iterations = x / 10, SampleSize = 10 }).ToArray(),
+      //parallel: true);
+    }
+
+    private static List<QuadraticAssignmentProblem> GetSomeRandomInstances(int dimension, int totalInstances, int seed) {
       var sync = new object();
-      var provider = new OneSizeInstanceProvider();
+      var provider = new OneSizeInstanceProvider(dimension);
       var instances = new List<QuadraticAssignmentProblem>();
       var random = new FastRandom(seed);
@@ -87 +107 @@
         }
       });
-      return instances;
-    }
-
-    private static HashSet<string> selectedInstances = new HashSet<string>() {
-      "bur26a", "chr25a", "dre24", "els19", "esc32a", "had20", "kra32", "lipa30a", "lipa30b",
-      "nug30", "rou20", "scr20", "sko42", "ste36a", "tai25a", "tai25b", "tho30", "wil50",
-      "RAND-S-6x6-36-25-AFFY-00_rand_rand_bl", "RAND-S-6x6-36-25-AFFY-00_rand_rand_ci"
-    };
-    private static List<QuadraticAssignmentProblem> Get20DifferentClasses() {
-      var sync = new object();
-
-      var qapProviders = ApplicationManager.Manager.GetInstances<ProblemInstanceProvider<QAPData>>().ToList();
-      var instances = new List<QuadraticAssignmentProblem>();
-      foreach (var provider in qapProviders) {
-        if (provider is TaillardQAPInstanceProvider) continue;
-        Parallel.ForEach(provider.GetDataDescriptors(), desc => {
-          if (!selectedInstances.Contains(desc.Name)) return;
-          //if (desc.Name == "esc16f") return;
-          var qapData = provider.LoadData(desc);
-          //if (qapData.Dimension < 15 || qapData.Dimension > 150) return;
-          var qap = new QuadraticAssignmentProblem();
-          qap.Load(qapData);
-          lock (sync) {
-            instances.Add(qap);
-          }
-        });
-      }
       return instances;
     }
@@ -155 +148 @@
         TotalCount = totalCount,
         ExactAverageRank = exactRank / (double)totalCount,
-        ClsAverageRank = clsRank / (double)totalCount
-      };
-    }
-
-    private static void PrintMatchResult(MatchResult result) {
-      Console.WriteLine("{0}\t{1}\t{2}\t{3:F2}\t{4:F2}",
-        result.ExactCount, result.ClsCount, result.TotalCount,
-        result.ExactAverageRank, result.ClsAverageRank);
+        ClsAverageRank = clsRank / (double)totalCount,
+        TrainingDescriptionEffort = training.Average(x => x.DescriptionEffort),
+        TestDescriptionEffort = test.Average(x => x.DescriptionEffort)
+      };
     }
 
@@ -168 +157 @@
       using (var iter = instances.GetEnumerator()) {
         if (!iter.MoveNext()) return;
-        Console.WriteLine(string.Join(";", new[] {"Name", "Cls", "Dimension"}
+        Console.WriteLine(string.Join(";", new[] {"Name", "Cls", "Dimension", "Effort"}
           .Concat(iter.Current != null ? iter.Current.FeatureNames : new [] { "(null)" })));
         do {
@@ -178 +167 @@
     private static void PrintInstanceLine(InstanceDescriptor instance) {
       Console.WriteLine(string.Join(";",
-        new[] {instance.Name, instance.Cls, instance.Dimension.ToString(CultureInfo.CurrentCulture)}
+        new[] {instance.Name, instance.Cls, instance.Dimension.ToString(CultureInfo.CurrentCulture), instance.DescriptionEffort.ToString(CultureInfo.CurrentCulture)}
           .Concat(instance.FeatureValues.Select(x => x.ToString(CultureInfo.CurrentCulture)))));
     }
 
-    private static void ExploreMatching(List<QuadraticAssignmentProblem> instances, InstanceExplorer[] trainingExplorers, InstanceExplorer[] testExporers, bool parallel = false) {
+    private static void ExploreMatching(StreamWriter writer, List<QuadraticAssignmentProblem> instances, InstanceExplorer[] trainingExplorers, InstanceExplorer[] testExporers, string info, bool parallel = false) {
       var sync = new object();
 
@@ -223 +212 @@
           var result = Compare(point.Training.Value.Item2, normalizedTest);
           lock (sync) {
-            Console.WriteLine("{0}\t{1}\t{2}\t{3:F2}\t{4}\t{5:F2}\t{6}",
-              point.Training.Key.Effort, point.Test.Key.Effort, result.ExactCount,
-              result.ExactAverageRank, result.ClsCount, result.ClsAverageRank, result.TotalCount);
+            string output = string.Format("{0}\t{1}\t{2}\t{3:F2}\t{4}\t{5:F2}\t{6}\t{7}\t{8}\t{9}",
+              info, point.Training.Key.Effort, point.Test.Key.Effort, result.ExactCount,
+              result.ExactAverageRank, result.ClsCount, result.ClsAverageRank, result.TotalCount,
+              result.TrainingDescriptionEffort, result.TestDescriptionEffort);
+            writer.WriteLine(output);
+            Console.WriteLine(output);
           }
         });
@@ -233 +225 @@
           point.Training.Value.Item1.Apply(normalizedTest);
           var result = Compare(point.Training.Value.Item2, normalizedTest);
-          Console.WriteLine("{0}\t{1}\t{2}\t{3:F2}\t{4}\t{5:F2}\t{6}",
-            point.Training.Key.Effort, point.Test.Key.Effort, result.ExactCount,
-            result.ExactAverageRank, result.ClsCount, result.ClsAverageRank, result.TotalCount);
+          string output = string.Format("{0}\t{1}\t{2}\t{3:F2}\t{4}\t{5:F2}\t{6}\t{7}\t{8}\t{9}",
+            info, point.Training.Key.Effort, point.Test.Key.Effort, result.ExactCount,
+            result.ExactAverageRank, result.ClsCount, result.ClsAverageRank, result.TotalCount,
+            result.TrainingDescriptionEffort, result.TestDescriptionEffort);
+          writer.WriteLine(output);
+          Console.WriteLine(output);
         }
       }
@@ -246 +241 @@
       public double ExactAverageRank { get; set; }
       public double ClsAverageRank { get; set; }
+
+      public double TrainingDescriptionEffort { get; set; }
+      public double TestDescriptionEffort { get; set; }
     }
   }