Free cookie consent management tool by TermsFeed Policy Generator

Changeset 14691 for branches


Ignore:
Timestamp:
02/23/17 10:42:58 (7 years ago)
Author:
abeham
Message:

#2457: working on identification of problem instances

Location:
branches/PerformanceComparison
Files:
4 added
15 edited
1 copied

Legend:

Unmodified
Added
Removed

  • branches/PerformanceComparison/HeuristicLab.Algorithms.MemPR/3.3/HeuristicLab.Algorithms.MemPR-3.3.csproj

    r14678 r14691  
    132132    <Reference Include="HeuristicLab.Problems.DataAnalysis-3.4">
    133133      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Problems.DataAnalysis-3.4.dll</HintPath>
     134      <Private>False</Private>
     135    </Reference>
     136    <Reference Include="HeuristicLab.Problems.Instances-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
     137      <SpecificVersion>False</SpecificVersion>
     138      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Problems.Instances-3.3.dll</HintPath>
     139      <Private>False</Private>
     140    </Reference>
     141    <Reference Include="HeuristicLab.Problems.QuadraticAssignment-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
     142      <SpecificVersion>False</SpecificVersion>
     143      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Problems.QuadraticAssignment-3.3.dll</HintPath>
    134144      <Private>False</Private>
    135145    </Reference>

  • branches/PerformanceComparison/HeuristicLab.Algorithms.MemPR/3.3/MemPRContext.cs

    r14690 r14691  
    469469
    470470    private TContext parent;
     471    protected TContext BaseContext {
     472      get { return parent;}
     473    }
    471474    public IExecutionContext Parent {
    472475      get { return parent; }

  • branches/PerformanceComparison/HeuristicLab.Algorithms.MemPR/3.3/Permutation/LocalSearch/ExhaustiveHillClimb.cs

    r14690 r14691  
    2828using HeuristicLab.Optimization;
    2929using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
     30using HeuristicLab.Problems.QuadraticAssignment;
    3031
    3132namespace HeuristicLab.Algorithms.MemPR.Permutation.LocalSearch {
    … …  
    5455      var quality = context.Solution.Fitness;
    5556      try {
     57
     58        var min = 0.0;
     59        var max = 1.0;
     60        var qap = context.Problem as QuadraticAssignmentProblem;
     61        if (qap != null) {
     62          min = qap.LowerBound.Value;
     63          max = qap.AverageQuality.Value;
     64        }
     65
    5666        var path = new List<Tuple<Encodings.PermutationEncoding.Permutation, double>>();
    57         path.Add(Tuple.Create((Encodings.PermutationEncoding.Permutation)context.Solution.Solution.Clone(), quality));
     67        path.Add(Tuple.Create((Encodings.PermutationEncoding.Permutation)context.Solution.Solution.Clone(), (quality - min) / (max - min)));
    5868
    5969        var result = Exhaustive.HillClimb(context.Random, context.Solution.Solution, quality,
    6070          context.Maximization, context.Evaluate, CancellationToken.None);
    61 
    62         Tuple<Encodings.PermutationEncoding.Permutation, double, int> last = null;
     71       
     72        var evaluations = 0;
    6373        foreach (var step in result) {
    64           path.Add(Tuple.Create((Encodings.PermutationEncoding.Permutation)step.Item1.Clone(), step.Item2));
    65           last = step;
     74          path.Add(Tuple.Create((Encodings.PermutationEncoding.Permutation)step.Item1.Clone(), (step.Item2 - min) / (max - min)));
     75          evaluations = step.Item3; // last one will be actual evaluations
    6676        }
    6777        context.LocalSearchPaths.AddPath(path);
    68         context.IncrementEvaluatedSolutions(last.Item3);
     78        context.IncrementEvaluatedSolutions(evaluations);
    6979        context.Iterations = path.Count - 2;
    7080      } finally {

  • branches/PerformanceComparison/HeuristicLab.Algorithms.MemPR/3.3/Permutation/LocalSearch/ExhaustiveHillClimbSubspace.cs

    r14690 r14691  
    2828using HeuristicLab.Optimization;
    2929using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
     30using HeuristicLab.Problems.QuadraticAssignment;
    3031
    3132namespace HeuristicLab.Algorithms.MemPR.Permutation.LocalSearch {
    … …  
    5455      var quality = context.Solution.Fitness;
    5556      try {
     57        var min = 0.0;
     58        var max = 1.0;
     59        var qap = context.Problem as QuadraticAssignmentProblem;
     60        if (qap != null) {
     61          min = qap.LowerBound.Value;
     62          max = qap.AverageQuality.Value;
     63        }
     64
    5665        var path = new List<Tuple<Encodings.PermutationEncoding.Permutation, double>>();
    57         path.Add(Tuple.Create((Encodings.PermutationEncoding.Permutation)context.Solution.Solution.Clone(), quality));
     66        path.Add(Tuple.Create((Encodings.PermutationEncoding.Permutation)context.Solution.Solution.Clone(), (quality - min) / (max - min)));
    5867
    5968        var result = Exhaustive.HillClimb(context.Random, context.Solution.Solution, quality,
    6069          context.Maximization, context.Evaluate, CancellationToken.None, context.Subspace.Subspace);
    6170
    62         Tuple<Encodings.PermutationEncoding.Permutation, double, int> last = null;
     71        var evaluations = 0;
    6372        foreach (var step in result) {
    64           path.Add(Tuple.Create((Encodings.PermutationEncoding.Permutation)step.Item1.Clone(), step.Item2));
    65           last = step;
     73          path.Add(Tuple.Create((Encodings.PermutationEncoding.Permutation)step.Item1.Clone(), (step.Item2 - min) / (max - min)));
     74          evaluations = step.Item3; // last one will be actual evaluations
    6675        }
    6776        context.LocalSearchPaths.AddPath(path);
    68         context.IncrementEvaluatedSolutions(last.Item3);
     77        context.IncrementEvaluatedSolutions(evaluations);
    6978        context.Iterations = path.Count - 2;
    7079      } finally {

  • branches/PerformanceComparison/HeuristicLab.Algorithms.MemPR/3.3/Permutation/PermutationMemPRContext.cs

    r14552 r14691  
    2323using System.Runtime.Remoting.Contexts;
    2424using HeuristicLab.Algorithms.MemPR.Interfaces;
     25using HeuristicLab.Analysis.FitnessLandscape;
    2526using HeuristicLab.Common;
    2627using HeuristicLab.Core;
    … …  
    6162  [Item("MemPR Solution Context (permutation)", "MemPR solution context for permutation encoded problems.")]
    6263  [StorableClass]
    63   public sealed class PermutationMemPRSolutionContext : MemPRSolutionContext<ISingleObjectiveHeuristicOptimizationProblem, Encodings.PermutationEncoding.Permutation, PermutationMemPRPopulationContext, PermutationMemPRSolutionContext>, IPermutationSubspaceContext {
     64  public sealed class PermutationMemPRSolutionContext : MemPRSolutionContext<ISingleObjectiveHeuristicOptimizationProblem, Encodings.PermutationEncoding.Permutation, PermutationMemPRPopulationContext, PermutationMemPRSolutionContext>, IPermutationSubspaceContext,
     65    ILocalSearchPathContext<Encodings.PermutationEncoding.Permutation> {
    6466
    6567    [Storable]
    … …  
    8789      return new PermutationMemPRSolutionContext(this, cloner);
    8890    }
     91
     92    public DirectedPath<Encodings.PermutationEncoding.Permutation> LocalSearchPaths {
     93      get { return BaseContext.LocalSearchPaths; }
     94    }
    8995  }
    9096}

  • branches/PerformanceComparison/HeuristicLab.Analysis.FitnessLandscape/3.3/HeuristicLab.Analysis.FitnessLandscape-3.3.csproj

    r14678 r14691  
    207207    <Compile Include="CharacteristicCalculator\RandomWalkCalculator.cs" />
    208208    <Compile Include="ProblemCharacteristicAnalysis\CharacteristicCalculator.cs" />
     209    <Compile Include="ProblemCharacteristicAnalysis\CurveAnalysis.cs" />
    209210    <Compile Include="ProblemCharacteristicAnalysis\DoubleMatrixCharacteristicCalculator.cs" />
     211    <Compile Include="ProblemCharacteristicAnalysis\PermutationPathAnalysis.cs" />
    210212    <Compile Include="ProblemCharacteristicAnalysis\QAP\QAPCharacteristicCalculator.cs" />
    211213    <Compile Include="ProblemCharacteristicAnalysis\QAP\QAPDirectedWalk.cs" />

  • branches/PerformanceComparison/HeuristicLab.Analysis.FitnessLandscape/3.3/ProblemCharacteristicAnalysis/QAP/QAPDirectedWalk.cs

    r14690 r14691  
    7979    private QAPDirectedWalk(QAPDirectedWalk original, Cloner cloner) : base(original, cloner) { }
    8080    public QAPDirectedWalk() {
    81       characteristics.AddRange(new[] { "Swap2.Sharpness", "Swap2.Bumpiness", "Swap2.Flatness", "Swap2.Steadiness" }
     81      characteristics.AddRange(new[] { "Swap2.Sharpness", "Swap2.Bumpiness", "Swap2.Flatness", /*"Swap2.Steadiness"*/ }
    8282        .Select(x => new StringValue(x)).ToList());
    8383      Parameters.Add(new FixedValueParameter<IntValue>("Paths", "The number of paths to explore (a path is a set of solutions that connect two randomly chosen solutions).", new IntValue(50)));
    … …  
    118118
    119119      var trajectories = Run(random, (QuadraticAssignmentProblem)Problem, permutations, BestImprovement).ToList();
    120       var firstDerivatives = trajectories.Select(path => ApproximateDerivative(path).ToList()).ToList();
    121       var secondDerivatives = firstDerivatives.Select(d1 => ApproximateDerivative(d1).ToList()).ToList();
     120      var result = PermutationPathAnalysis.GetCharacteristics(trajectories);
    122121     
    123       var props = GetCharacteristics(trajectories, firstDerivatives, secondDerivatives).ToDictionary(x => x.Item1, x => x.Item2);
    124122      foreach (var chara in characteristics.CheckedItems.Select(x => x.Value.Value)) {
    125         if (chara == "Swap2.Sharpness") yield return new Result("Swap2.Sharpness", new DoubleValue(props["Sharpness"]));
    126         if (chara == "Swap2.Bumpiness") yield return new Result("Swap2.Bumpiness", new DoubleValue(props["Bumpiness"]));
    127         if (chara == "Swap2.Flatness") yield return new Result("Swap2.Flatness", new DoubleValue(props["Flatness"]));
    128         if (chara == "Swap2.Steadiness") yield return new Result("Swap2.Steadiness", new DoubleValue(props["Steadiness"]));
    129       }
    130     }
    131 
    132     public static IEnumerable<IResult> Calculate(List<List<Tuple<Permutation, double>>> trajectories) {
    133       var firstDerivatives = trajectories.Select(path => ApproximateDerivative(path).ToList()).ToList();
    134       var secondDerivatives = firstDerivatives.Select(d1 => ApproximateDerivative(d1).ToList()).ToList();
    135 
    136       var props = GetCharacteristics(trajectories, firstDerivatives, secondDerivatives).ToDictionary(x => x.Item1, x => x.Item2);
    137       yield return new Result("Swap2.Sharpness", new DoubleValue(props["Sharpness"]));
    138       yield return new Result("Swap2.Bumpiness", new DoubleValue(props["Bumpiness"]));
    139       yield return new Result("Swap2.Flatness", new DoubleValue(props["Flatness"]));
    140       yield return new Result("Swap2.Steadiness", new DoubleValue(props["Steadiness"]));
     123        if (chara == "Swap2.Sharpness") yield return new Result("Swap2.Sharpness", new DoubleValue(result.Sharpness));
     124        if (chara == "Swap2.Bumpiness") yield return new Result("Swap2.Bumpiness", new DoubleValue(result.Bumpiness));
     125        if (chara == "Swap2.Flatness") yield return new Result("Swap2.Flatness", new DoubleValue(result.Flatness));
     126      }
    141127    }
    142128
    … …  
    158144    }
    159145
    160     private static IEnumerable<Tuple<string, double>> GetCharacteristics(List<List<Tuple<Permutation, double>>> f, List<List<Tuple<Permutation, double>>> f1, List<List<Tuple<Permutation, double>>> f2) {
    161       var sharpness = f2.Average(x => Area(x));
    162       var bumpiness = 0.0;
    163       var flatness = 0.0;
    164       var downPointing = f1.Where(x => x.Min(y => y.Item2) < 0).ToList();
    165 
    166       var steadiness = 0.0;
    167       foreach (var path in downPointing) {
    168         steadiness += ComBelowZero(path);
    169       }
    170       if (downPointing.Count > 0) steadiness /= downPointing.Count;
    171 
    172       for (var p = 0; p < f2.Count; p++) {
    173         if (f2[p].Count <= 2) continue;
    174         var bump = 0;
    175         var flat = 0;
    176         for (var i = 0; i < f2[p].Count - 1; i++) {
    177           if ((f2[p][i].Item2 > 0 && f2[p][i + 1].Item2 < 0) || (f2[p][i].Item2 < 0 && f2[p][i + 1].Item2 > 0)) {
    178             bump++;
    179           } else if (f2[p][i].Item2 == 0) {
    180             flat++;
    181           }
    182         }
    183         bumpiness += bump / (f2[p].Count - 1.0);
    184         flatness += flat / (f2[p].Count - 1.0);
    185       }
    186       bumpiness /= f2.Count;
    187       flatness /= f2.Count;
    188       return new[] {
    189       Tuple.Create("Sharpness", sharpness),
    190       Tuple.Create("Bumpiness", bumpiness),
    191       Tuple.Create("Flatness", flatness),
    192       Tuple.Create("Steadiness", steadiness)
    193     };
    194     }
    195 
    196     public static IEnumerable<Tuple<Permutation, double>> BestDirectedWalk(QuadraticAssignmentProblem qap, Permutation start, Permutation end) {
     146    private static IEnumerable<Tuple<Permutation, double>> BestDirectedWalk(QuadraticAssignmentProblem qap, Permutation start, Permutation end) {
    197147      var N = qap.Weights.Rows;
    198148      var sol = start;
    … …  
    225175    }
    226176
    227     public static IEnumerable<Tuple<Permutation, double>> FirstDirectedWalk(IRandom random, QuadraticAssignmentProblem qap, Permutation start, Permutation end) {
     177    private static IEnumerable<Tuple<Permutation, double>> FirstDirectedWalk(IRandom random, QuadraticAssignmentProblem qap, Permutation start, Permutation end) {
    228178      var N = qap.Weights.Rows;
    229179      var sol = start;
    … …  
    260210    }
    261211
    262     private static double Area(IEnumerable<Tuple<Permutation, double>> path) {
    263       var iter = path.GetEnumerator();
    264       if (!iter.MoveNext()) return 0.0;
    265       var area = 0.0;
    266       var prev = iter.Current;
    267       while (iter.MoveNext()) {
    268         area += TrapezoidArea(prev, iter.Current);
    269         prev = iter.Current;
    270       }
    271       return area;
    272     }
    273 
    274     private static double TrapezoidArea(Tuple<Permutation, double> a, Tuple<Permutation, double> b) {
    275       var area = 0.0;
    276       var dist = Dist(a.Item1, b.Item1);
    277       if ((a.Item2 <= 0 && b.Item2 <= 0) || (a.Item2 >= 0 && b.Item2 >= 0))
    278         area += dist * (Math.Abs(a.Item2) + Math.Abs(b.Item2)) / 2.0;
    279       else {
    280         var k = (b.Item2 - a.Item2) / dist;
    281         var d = a.Item2;
    282         var x = -d / k;
    283         area += Math.Abs(x * a.Item2 / 2.0);
    284         area += Math.Abs((dist - x) * b.Item2 / 2.0);
    285       }
    286       return area;
    287     }
    288 
    289     // Center-of-Mass
    290     private static double ComBelowZero(IEnumerable<Tuple<Permutation, double>> path) {
    291       var area = 0.0;
    292       var com = 0.0;
    293       var nwalkDist = 0.0;
    294       Tuple<Permutation, double> prev = null;
    295       var iter = path.GetEnumerator();
    296       while (iter.MoveNext()) {
    297         var c = iter.Current;
    298         if (prev != null) {
    299           var ndist = Dist(prev.Item1, c.Item1) / (double)c.Item1.Length;
    300           nwalkDist += ndist;
    301           if (prev.Item2 < 0 || c.Item2 < 0) {
    302             var a = TrapezoidArea(prev, c) / (double)c.Item1.Length;
    303             area += a;
    304             com += (nwalkDist - (ndist / 2.0)) * a;
    305           }
    306         }
    307         prev = c;
    308       }
    309       return com / area;
    310     }
    311 
    312     private static IEnumerable<Tuple<Permutation, double>> ApproximateDerivative(IEnumerable<Tuple<Permutation, double>> data) {
    313       Tuple<Permutation, double> prev = null, prev2 = null;
    314       foreach (var d in data) {
    315         if (prev == null) {
    316           prev = d;
    317           continue;
    318         }
    319         if (prev2 == null) {
    320           prev2 = prev;
    321           prev = d;
    322           continue;
    323         }
    324         var dist = Dist(prev2.Item1, d.Item1);
    325         yield return Tuple.Create(prev.Item1, (d.Item2 - prev2.Item2) / (double)dist);
    326         prev2 = prev;
    327         prev = d;
    328       }
    329     }
    330 
    331     private static double Dist(Permutation a, Permutation b) {
    332       var dist = 0;
    333       for (var i = 0; i < a.Length; i++)
    334         if (a[i] != b[i]) dist++;
    335       return dist;
    336     }
    337 
    338212    private static int[] GetInverse(Permutation p) {
    339213      var inv = new int[p.Length];

  • branches/PerformanceComparison/HeuristicLab.Analysis.FitnessLandscape/3.3/ProblemInstanceAnalysis/ProblemInstanceAnalyzer.cs

    r14678 r14691  
    7878      if (currentCharacteristics == null) return base.Apply();
    7979
    80       var order = Enumerable.Range(0, kbCharacteristics.Rows)
     80      var means = kbCharacteristics.GetRow(kbCharacteristics.Rows - 2).ToArray();
     81      var stdevs = kbCharacteristics.GetRow(kbCharacteristics.Rows - 1).ToArray();
     82
     83      for (var i = 0; i < means.Length; i++) {
     84        currentCharacteristics[i] = (currentCharacteristics[i] - means[i]) / stdevs[i];
     85      }
     86
     87      var order = Enumerable.Range(0, kbCharacteristics.Rows - 2)
    8188        .Select(row => new { Row = row, MSE = kbCharacteristics.GetRow(row).Zip(currentCharacteristics, (a, b) => (a - b) * (a - b)).Average() })
    8289        .OrderBy(x => x.MSE);
    8390
    8491      var instances = kbCharacteristics.RowNames.ToList();
    85       while (instances.Count < kbCharacteristics.Rows)
     92      while (instances.Count < kbCharacteristics.Rows - 2)
    8693        instances.Add(instances.Count.ToString(CultureInfo.CurrentCulture.NumberFormat));
    8794

  • branches/PerformanceComparison/HeuristicLab.Analysis.FitnessLandscape/3.3/ProblemInstanceAnalysis/QAPPRProblemInstanceAnalyzer.cs

    r14678 r14691  
    4949      if (trajectories.Count == 0) return null;
    5050
    51       var characteristics = QAPDirectedWalk.Calculate(trajectories).Select(x => x.Value).ToList();
    52       var result = new DoubleArray(characteristics.Count);
    53       for (var i = 0; i < characteristics.Count; i++) {
    54         var dv = characteristics[i] as DoubleValue;
    55         if (dv != null) result[i] = dv.Value;
    56         else {
    57           var iv = characteristics[i] as IntValue;
    58           if (iv != null) result[i] = iv.Value;
    59         }
    60       }
    61       return result;
     51      return new DoubleArray(PermutationPathAnalysis.GetCharacteristics(trajectories).GetValues().ToArray());
    6252    }
    6353  }

  • branches/PerformanceComparison/HeuristicLab.Optimization.Views

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/PerformanceComparison/HeuristicLab.Problems.Instances.QAPLIB/3.3/HeuristicLab.Problems.Instances.QAPLIB-3.3.csproj

    r11650 r14691  
    1919    <DebugType>full</DebugType>
    2020    <Optimize>false</Optimize>
    21     <OutputPath>..\..\bin\</OutputPath>
     21    <OutputPath>..\..\..\..\trunk\sources\bin\</OutputPath>
    2222    <DefineConstants>DEBUG;TRACE</DefineConstants>
    2323    <ErrorReport>prompt</ErrorReport>
    … …  
    2828    <DebugType>pdbonly</DebugType>
    2929    <Optimize>true</Optimize>
    30     <OutputPath>..\..\bin\</OutputPath>
     30    <OutputPath>..\..\..\..\trunk\sources\bin\</OutputPath>
    3131    <DefineConstants>TRACE</DefineConstants>
    3232    <ErrorReport>prompt</ErrorReport>
    … …  
    4242  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
    4343    <DebugSymbols>true</DebugSymbols>
    44     <OutputPath>..\..\bin\</OutputPath>
     44    <OutputPath>..\..\..\..\trunk\sources\bin\</OutputPath>
    4545    <DefineConstants>DEBUG;TRACE</DefineConstants>
    4646    <DebugType>full</DebugType>
    … …  
    5858  </PropertyGroup>
    5959  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
    60     <OutputPath>..\..\bin\</OutputPath>
     60    <OutputPath>..\..\..\..\trunk\sources\bin\</OutputPath>
    6161    <DefineConstants>TRACE</DefineConstants>
    6262    <Optimize>true</Optimize>
    … …  
    7676  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x86'">
    7777    <DebugSymbols>true</DebugSymbols>
    78     <OutputPath>..\..\bin\</OutputPath>
     78    <OutputPath>..\..\..\..\trunk\sources\bin\</OutputPath>
    7979    <DefineConstants>DEBUG;TRACE</DefineConstants>
    8080    <DebugType>full</DebugType>
    … …  
    9292  </PropertyGroup>
    9393  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x86'">
    94     <OutputPath>..\..\bin\</OutputPath>
     94    <OutputPath>..\..\..\..\trunk\sources\bin\</OutputPath>
    9595    <DefineConstants>TRACE</DefineConstants>
    9696    <Optimize>true</Optimize>
    … …  
    109109  </PropertyGroup>
    110110  <ItemGroup>
     111    <Reference Include="HeuristicLab.Common-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
     112      <SpecificVersion>False</SpecificVersion>
     113      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Common-3.3.dll</HintPath>
     114      <Private>False</Private>
     115    </Reference>
     116    <Reference Include="HeuristicLab.PluginInfrastructure-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
     117      <SpecificVersion>False</SpecificVersion>
     118      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.PluginInfrastructure-3.3.dll</HintPath>
     119      <Private>False</Private>
     120    </Reference>
     121    <Reference Include="HeuristicLab.Problems.Instances-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
     122      <SpecificVersion>False</SpecificVersion>
     123      <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Problems.Instances-3.3.dll</HintPath>
     124      <Private>False</Private>
     125    </Reference>
    111126    <Reference Include="System" />
    112127    <Reference Include="System.Core" />
    … …  
    115130  </ItemGroup>
    116131  <ItemGroup>
     132    <Compile Include="OneSizeDataDescriptor.cs" />
     133    <Compile Include="OneSizeInstanceProvider.cs" />
    117134    <Compile Include="TaillardQAPInstanceProvider.cs" />
    118135    <Compile Include="DreznerQAPInstanceProvider.cs" />
    … …  
    137154  <ItemGroup>
    138155    <None Include="HeuristicLab.snk" />
    139   </ItemGroup>
    140   <ItemGroup>
    141     <ProjectReference Include="..\..\HeuristicLab.Common\3.3\HeuristicLab.Common-3.3.csproj">
    142       <Project>{A9AD58B9-3EF9-4CC1-97E5-8D909039FF5C}</Project>
    143       <Name>HeuristicLab.Common-3.3</Name>
    144       <Private>False</Private>
    145     </ProjectReference>
    146     <ProjectReference Include="..\..\HeuristicLab.PluginInfrastructure\3.3\HeuristicLab.PluginInfrastructure-3.3.csproj">
    147       <Project>{94186A6A-5176-4402-AE83-886557B53CCA}</Project>
    148       <Name>HeuristicLab.PluginInfrastructure-3.3</Name>
    149       <Private>False</Private>
    150     </ProjectReference>
    151     <ProjectReference Include="..\..\HeuristicLab.Problems.Instances\3.3\HeuristicLab.Problems.Instances-3.3.csproj">
    152       <Project>{3540E29E-4793-49E7-8EE2-FEA7F61C3994}</Project>
    153       <Name>HeuristicLab.Problems.Instances-3.3</Name>
    154       <Private>False</Private>
    155     </ProjectReference>
    156156  </ItemGroup>
    157157  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />

  • branches/PerformanceComparison/PerformanceComparison.sln

    r14678 r14691  
    2525EndProject
    2626Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Algorithms.DataAnalysis-3.4", "HeuristicLab.Algorithms.DataAnalysis\3.4\HeuristicLab.Algorithms.DataAnalysis-3.4.csproj", "{2E782078-FA81-4B70-B56F-74CE38DAC6C8}"
     27EndProject
     28Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Problems.Instances.QAPLIB-3.3", "HeuristicLab.Problems.Instances.QAPLIB\3.3\HeuristicLab.Problems.Instances.QAPLIB-3.3.csproj", "{73F29D43-5714-4069-8FAB-0D18FEB5F175}"
    2729EndProject
    2830Global
    … …  
    168170    {2E782078-FA81-4B70-B56F-74CE38DAC6C8}.Release|x86.ActiveCfg = Release|x86
    169171    {2E782078-FA81-4B70-B56F-74CE38DAC6C8}.Release|x86.Build.0 = Release|x86
     172    {73F29D43-5714-4069-8FAB-0D18FEB5F175}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
     173    {73F29D43-5714-4069-8FAB-0D18FEB5F175}.Debug|Any CPU.Build.0 = Debug|Any CPU
     174    {73F29D43-5714-4069-8FAB-0D18FEB5F175}.Debug|x64.ActiveCfg = Debug|x64
     175    {73F29D43-5714-4069-8FAB-0D18FEB5F175}.Debug|x64.Build.0 = Debug|x64
     176    {73F29D43-5714-4069-8FAB-0D18FEB5F175}.Debug|x86.ActiveCfg = Debug|x86
     177    {73F29D43-5714-4069-8FAB-0D18FEB5F175}.Debug|x86.Build.0 = Debug|x86
     178    {73F29D43-5714-4069-8FAB-0D18FEB5F175}.Release|Any CPU.ActiveCfg = Release|Any CPU
     179    {73F29D43-5714-4069-8FAB-0D18FEB5F175}.Release|Any CPU.Build.0 = Release|Any CPU
     180    {73F29D43-5714-4069-8FAB-0D18FEB5F175}.Release|x64.ActiveCfg = Release|x64
     181    {73F29D43-5714-4069-8FAB-0D18FEB5F175}.Release|x64.Build.0 = Release|x64
     182    {73F29D43-5714-4069-8FAB-0D18FEB5F175}.Release|x86.ActiveCfg = Release|x86
     183    {73F29D43-5714-4069-8FAB-0D18FEB5F175}.Release|x86.Build.0 = Release|x86
    170184  EndGlobalSection
    171185  GlobalSection(SolutionProperties) = preSolution

  • branches/PerformanceComparison/ProblemInstanceIdentifier/InstanceDescriptor.cs

    r14690 r14691  
    4646
    4747    public static InstanceDescriptor FromPaths(QuadraticAssignmentProblem qap, List<List<Tuple<Permutation, double>>> trajectories) {
    48       var features = QAPDirectedWalk.Calculate(trajectories).ToDictionary(x => x.Name, x => x.Value);
    49      
     48      var result = PermutationPathAnalysis.GetCharacteristics(trajectories);
     49
    5050      return new InstanceDescriptor() {
    5151        Name = qap.Name,
    5252        Cls = GetClass(qap.Name),
    5353        Dimension = qap.Weights.Rows,
    54         FeatureNames = features.Keys.ToArray(),
    55         FeatureValues = features.Values.Select(x => ((DoubleValue)x).Value).ToArray()
     54        FeatureNames = result.GetNames(),
     55        FeatureValues = result.GetValues()
    5656      };
    5757    }
    … …  
    8787    private double[] featureStdevs;
    8888
     89    public IEnumerable<double> GetMeans() {
     90      return featureMeans;
     91    }
     92    public IEnumerable<double> GetStdevs() {
     93      return featureStdevs;
     94    }
     95
    8996    private InstancesStandardizer() { }
    9097
    … …  
    101108      return standardizer;
    102109    }
     110    public static InstancesStandardizer CreateAndApply(IList<InstanceDescriptor> instances) {
     111      var standardizer = Create(instances);
     112      standardizer.Apply(instances);
     113      return standardizer;
     114    }
    103115
    104116    public void Apply(IList<InstanceDescriptor> instances) {

  • branches/PerformanceComparison/ProblemInstanceIdentifier/InstanceExplorer.cs

    r14690 r14691  
    1 using System.Linq;
     1using System;
     2using System.Linq;
    23using System.Threading;
     4using HeuristicLab.Algorithms.MemPR.Permutation;
    35using HeuristicLab.Analysis.FitnessLandscape;
    46using HeuristicLab.Data;
    … …  
    5759      var walk = new RandomWalk() {
    5860        SeedParameter = { Value = { Value = seed ?? 0 } },
    59         SetSeedRandomlyParameter = { Value = { Value = seed.HasValue } },
     61        SetSeedRandomlyParameter = { Value = { Value = !seed.HasValue } },
    6062        MaximumIterationsParameter = { Value = { Value = Iterations } },
    6163        RepetitionsParameter = { Value = { Value = 1 } }
    … …  
    7173    }
    7274  }
     75
     76  public class MemPRExplorer : InstanceExplorer {
     77    public int Seconds { get; set; }
     78
     79    public bool IncludeLocalSearch { get; set; }
     80
     81    public override string Name { get { return "MemPR Explorer"; } }
     82    public override int Effort { get { return Seconds; } }
     83
     84    public MemPRExplorer() {
     85
     86    }
     87
     88    public override InstanceDescriptor Explore(QuadraticAssignmentProblem problem, int? seed = null) {
     89      var memPr = new PermutationMemPR();
     90      memPr.Problem = problem;
     91      memPr.Prepare(true);
     92      memPr.MaximumExecutionTime = TimeSpan.FromSeconds(Seconds);
     93      memPr.SetSeedRandomly = !seed.HasValue;
     94      memPr.Seed = seed ?? 0;
     95      memPr.StartSync();
     96      if (memPr.Context.RelinkedPaths.IsEmpty
     97        || IncludeLocalSearch && memPr.Context.LocalSearchPaths.IsEmpty) {
     98        Console.Write("{0} not all paths present!", problem.Name);
     99        return null;
     100      };
     101
     102      var features = PermutationPathAnalysis.GetCharacteristics(memPr.Context.RelinkedPaths.Paths.ToList());
     103      var result = features.GetValues();
     104      var resultNames = features.GetNames();
     105      if (IncludeLocalSearch) {
     106        features = PermutationPathAnalysis.GetCharacteristics(memPr.Context.LocalSearchPaths.Paths.ToList());
     107        result = result.Concat(features.GetValues()).ToArray();
     108        resultNames = resultNames.Concat(features.GetNames()).ToArray();
     109      }
     110      return new InstanceDescriptor(problem.Name, InstanceDescriptor.GetClass(problem.Name), problem.Weights.Rows, resultNames, result);
     111    }
     112  }
    73113}

  • branches/PerformanceComparison/ProblemInstanceIdentifier/Program.cs

    r14690 r14691  
    99using HeuristicLab.Problems.Instances.QAPLIB;
    1010using HeuristicLab.Problems.QuadraticAssignment;
     11using HeuristicLab.Random;
    1112
    1213namespace ProblemInstanceIdentifier {
    … …  
    1819    };
    1920    static void Main(string[] args) {
     21      var instances = Get20DifferentClasses();
     22      //var instances = GetSomeRandomInstances(50);
     23
     24      /*var classes = instances.Select(InstanceDescriptor.FromProblemOnly).GroupBy(x => x.Cls);
     25      foreach (var cls in classes.OrderBy(x => x.Key)) {
     26        Console.WriteLine("{0};{1}", cls.Key, cls.Count());
     27      }*/
     28
     29      var prExplorer = new PathRelinkingExplorer() {
     30        LocalOptima = false,
     31        Paths = 200
     32      };
     33      var prLocalExplorer = new PathRelinkingExplorer() {
     34        LocalOptima = true,
     35        Paths = 200
     36      };
     37      var memPrExplorer = new MemPRExplorer() {
     38        IncludeLocalSearch = false,
     39        Seconds = 10
     40      };
     41
     42      var training = GenerateData(instances, prExplorer, parallel:true);
     43      var standardizer = InstancesStandardizer.CreateAndApply(training);
     44      var test = GenerateData(instances, prExplorer, parallel: false);
     45      standardizer.Apply(test);
     46      PrintMatchResult(Compare(training, test));
     47
     48      ExploreMatching(instances, new [] { prLocalExplorer }, new [] { memPrExplorer });
     49    }
     50
     51    private static List<QuadraticAssignmentProblem> GetSomeRandomInstances(int totalInstances) {
     52      var sync = new object();
     53      var provider = new OneSizeInstanceProvider();
     54      var instances = new List<QuadraticAssignmentProblem>();
     55      var random = new FastRandom(0);
     56      Parallel.ForEach(provider.GetDataDescriptors().Shuffle(random), desc => {
     57        var qapData = provider.LoadData(desc);
     58        if (qapData.Dimension < 25) return;
     59        if (instances.Count >= totalInstances) return;
     60        var qap = new QuadraticAssignmentProblem();
     61        qap.Load(qapData);
     62        lock (sync) {
     63          if (instances.Count >= totalInstances) return;
     64          instances.Add(qap);
     65        }
     66      });
     67      return instances;
     68    }
     69
     70    private static List<QuadraticAssignmentProblem> Get20DifferentClasses() {
    2071      var sync = new object();
    2172
    … …  
    3687        });
    3788      }
    38 
    39       /*var classes = instances.Select(InstanceDescriptor.FromProblemOnly).GroupBy(x => x.Cls);
    40       foreach (var cls in classes.OrderBy(x => x.Key)) {
    41         Console.WriteLine("{0};{1}", cls.Key, cls.Count());
    42       }*/
    43 
    44       //var kb = GenerateKnowledgeBase(instances, 200, localOptima: false);
    45 
    46 
    47       //var paths = new[] { 1, 2, 3, 5, 10, 20, 30, 50, 100, 200 };
    48       //var kbExplorers = paths.Select(x => new PathRelinkingExplorer() { Paths = x }).ToArray();
    49       //var expExplorers = paths.Select(x => new PathRelinkingExplorer() { Paths = x }).ToArray();
    50       //ExploreSelfIdentification(instances, kbExplorers, expExplorers);
    51 
    52       var iterations = new[] { 100, 1000, 10000, 100000 };
    53       var kbExplorers = iterations.Select(x => new RandomWalkExplorer() { Iterations = x }).ToArray();
    54       var expExplorers = iterations.Select(x => new RandomWalkExplorer() { Iterations = x }).ToArray();
    55       ExploreSelfIdentification(instances, kbExplorers, expExplorers);
    56 
    57       //ExploreMemPrIdentification(instances);
    58     }
    59 
    60     private static List<InstanceDescriptor> GenerateKnowledgeBase(List<QuadraticAssignmentProblem> instances, InstanceExplorer explorer) {
    61       var headerPrinted = false;
    62       var sync = new object();
    63       var kb = new List<InstanceDescriptor>();
    64       Parallel.ForEach(instances.OrderBy(x => x.Weights.Rows), qap => {
     89      return instances;
     90    }
     91
     92    private static List<InstanceDescriptor> GenerateData(List<QuadraticAssignmentProblem> instances, InstanceExplorer explorer, bool parallel = false) {
     93      var sync = new object();
     94      var data = new List<InstanceDescriptor>();
     95      Action<QuadraticAssignmentProblem> body = (qap) => {
    6596        var instance = explorer.Explore(qap);
    66         lock (sync) {
    67           if (!headerPrinted) {
    68             headerPrinted = true;
    69             Console.WriteLine(string.Join(";",
    70               new [] { "Name", "Cls", "Dimension" }
    71               .Concat(instance.FeatureNames)));
    72           }
    73           PrintInstanceLine(instance);
    74           kb.Add(instance);
    75         }
    76       });
    77       var standardizer = InstancesStandardizer.Create(kb);
    78       var normalizedKb = kb.Select(x => new InstanceDescriptor(x)).ToList();
    79       standardizer.Apply(normalizedKb);
    80       Console.WriteLine();
    81       foreach (var instance in kb) {
    82         PrintInstanceLine(instance);
    83       }
    84       //return normalizedKb;
    85       return kb;
     97        if (instance == null) return;
     98        lock (sync) {
     99          data.Add(instance);
     100        }
     101      };
     102      if (parallel) {
     103        Parallel.ForEach(instances, body);
     104      } else {
     105        foreach (var qap in instances) body(qap);
     106      }
     107      return data;
     108    }
     109
     110    private static MatchResult Compare(List<InstanceDescriptor> training, List<InstanceDescriptor> test) {
     111      int exactCount = 0, clsCount = 0, totalCount = 0;
     112      int exactRank = 0, clsRank = 0;
     113      foreach (var e in test) {
     114        var ordered = training.OrderBy(x => x.CalculateSimilarity(e)).ToList();
     115        var bestMatch = ordered.First();
     116        if (bestMatch.Cls == e.Cls) clsCount++;
     117        if (bestMatch.Name == e.Name) exactCount++;
     118        totalCount++;
     119        clsRank += ordered.FindIndex((id) => id.Cls == e.Cls) + 1;
     120        exactRank += ordered.FindIndex((id) => id.Name == e.Name) + 1;
     121      }
     122
     123      return new MatchResult() {
     124        ExactCount = exactCount,
     125        ClsCount = clsCount,
     126        TotalCount = totalCount,
     127        ExactAverageRank = exactRank / (double)totalCount,
     128        ClsAverageRank = clsRank / (double)totalCount
     129      };
     130    }
     131
     132    private static void PrintMatchResult(MatchResult result) {
     133      Console.WriteLine("{0}\t{1}\t{2}\t{3:F2}\t{4:F2}",
     134        result.ExactCount, result.ClsCount, result.TotalCount,
     135        result.ExactAverageRank, result.ClsAverageRank);
     136    }
     137
     138    private static void PrintData(List<InstanceDescriptor> instances) {
     139      using (var iter = instances.GetEnumerator()) {
     140        if (!iter.MoveNext()) return;
     141        Console.WriteLine(string.Join(";", new[] {"Name", "Cls", "Dimension"}
     142          .Concat(iter.Current != null ? iter.Current.FeatureNames : new [] { "(null)" })));
     143        do {
     144          PrintInstanceLine(iter.Current);
     145        } while (iter.MoveNext());
     146      }
    86147    }
    87148
    … …  
    92153    }
    93154
    94     private static void ExploreSelfIdentification(List<QuadraticAssignmentProblem> instances, InstanceExplorer[] kbExplorer, InstanceExplorer[] expExporer) {
    95       var sync = new object();
    96       Console.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}\t{5}", "Repetition", "KB-Effort", "Exp-Effort",
    97         "Exact-Hits", "No-Hits", "Exact-Rank");
    98       var kbSeeds = Enumerable.Range(0, 20).ToList();
    99       var expSeeds = Enumerable.Range(20, 20).ToList();
    100       for (var r = 0; r < 10; r++) {
    101         var rlokal = r;
    102         Parallel.ForEach(kbExplorer, kbPaths => {
    103         //foreach (var kbPaths in kbExplorer) {
    104           foreach (var expPaths in expExporer) {
    105             //if (expPaths > kbPaths) continue;
    106             var kb = new List<InstanceDescriptor>();
    107             var exp = new List<InstanceDescriptor>();
    108             foreach (var qap in instances) {
    109               kb.Add(kbPaths.Explore(qap, kbSeeds[rlokal]));
    110               exp.Add(expPaths.Explore(qap, expSeeds[rlokal]));
    111             }
    112             var standardizer = InstancesStandardizer.Create(kb);
    113             standardizer.Apply(kb);
    114             standardizer.Apply(exp);
    115             int exactCount = 0, clsCount = 0, missedCount = 0;
    116             int exactRank = 0, clsRank = 0;
    117             foreach (var e in exp) {
    118               var ordered = kb.OrderBy(x => x.CalculateSimilarity(e)).ToList();
    119               var bestMatch = ordered.First();
    120               if (bestMatch.Cls == e.Cls) {
    121                 clsCount++;
    122                 if (bestMatch.Name == e.Name) exactCount++;
    123               }
    124               else missedCount++;
    125               clsRank += ordered.FindIndex((id) => id.Cls == e.Cls) + 1;
    126               exactRank += ordered.FindIndex((id) => id.Name == e.Name) + 1;
    127             }
    128             lock (sync) {
    129               Console.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}\t{5:F2}", rlokal, kbPaths.Effort, expPaths.Effort, exactCount,
    130                 missedCount, exactRank / (double) exp.Count);
    131             }
     155    private static void ExploreMatching(List<QuadraticAssignmentProblem> instances, InstanceExplorer[] trainingExplorers, InstanceExplorer[] testExporers, bool parallel = false) {
     156      var sync = new object();
     157      var rand = new Random();
     158      var first = rand.Next();
     159      var second = rand.Next();
     160      while (first == second) second = rand.Next();
     161
     162      var knowledgeBase = new Dictionary<InstanceExplorer, Tuple<InstancesStandardizer, List<InstanceDescriptor>>>();
     163      Action<InstanceExplorer> trainingBody = (kbExplorer) => {
     164        var trainingData = GenerateData(instances, kbExplorer, parallel);
     165        var standardizer = InstancesStandardizer.Create(trainingData);
     166        standardizer.Apply(trainingData);
     167        lock (sync) {
     168          knowledgeBase.Add(kbExplorer, Tuple.Create(standardizer, trainingData));
     169        }
     170      };
     171      if (parallel) {
     172        Parallel.ForEach(trainingExplorers, trainingBody);
     173      } else {
     174        foreach (var kbExplorer in trainingExplorers) trainingBody(kbExplorer);
     175      }
     176
     177      var experimentBase = new Dictionary<InstanceExplorer, List<InstanceDescriptor>>();
     178      Action<InstanceExplorer> testBody = (expExplorer) => {
     179        var trainingData = GenerateData(instances, expExplorer, parallel);
     180        lock (sync) {
     181          experimentBase.Add(expExplorer, trainingData);
     182        }
     183      };
     184      if (parallel) {
     185        Parallel.ForEach(testExporers, testBody);
     186      } else {
     187        foreach (var expExplorer in testExporers) testBody(expExplorer);
     188      }
     189
     190      var data = from kb in knowledgeBase
     191        from exp in experimentBase
     192        select new { Training = kb, Test = exp };
     193
     194      if (parallel) {
     195        Parallel.ForEach(data, (point) => {
     196          var normalizedTest = point.Test.Value.Select(x => new InstanceDescriptor(x)).ToList();
     197          point.Training.Value.Item1.Apply(normalizedTest);
     198          var result = Compare(point.Training.Value.Item2, normalizedTest);
     199          lock (sync) {
     200            Console.WriteLine("{0}\t{1}\t{2}\t{3:F2}\t{4}\t{5:F2}\t{6}",
     201              point.Training.Key.Effort, point.Test.Key.Effort, result.ExactCount,
     202              result.ExactAverageRank, result.ClsCount, result.ClsAverageRank, result.TotalCount);
    132203          }
    133204        });
    134         //}
    135       }
    136     }
    137    
    138     private static void ExploreMemPrIdentification(List<QuadraticAssignmentProblem> instances) {
    139       var sync = new object();
    140       Console.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}\t{5}", "Repetition", "KB-Runtime", "Exp-Runtime",
    141         "Exact-Hits", "No-Hits", "Average-Rank");
    142       var paths = new[] { 10 };
    143       var repetitions = 10;
    144       var kbSeeds = Enumerable.Range(0, repetitions).ToList();
    145       var expSeeds = Enumerable.Range(repetitions, repetitions).ToList();
    146       for (var r = 0; r < repetitions; r++) {
    147         var rlokal = r;
    148         Parallel.ForEach(paths, kbPaths => {
    149           var memPr = new PermutationMemPR();
    150           foreach (var expPaths in paths) {
    151             //if (expPaths > kbPaths) continue;
    152             var kb = new List<InstanceDescriptor>();
    153             var exp = new List<InstanceDescriptor>();
    154             foreach (var qap in instances.Select(x => (QuadraticAssignmentProblem)x.Clone())) {
    155               memPr.Problem = qap;
    156               memPr.Prepare(true);
    157               memPr.Seed = kbSeeds[rlokal];
    158               memPr.MaximumExecutionTime = TimeSpan.FromSeconds(kbPaths);
    159               memPr.StartSync();
    160               if (memPr.Context.RelinkedPaths.IsEmpty) continue;
    161               kb.Add(InstanceDescriptor.FromPaths(qap, memPr.Context.RelinkedPaths.Paths.ToList()));
    162               memPr.Prepare(true);
    163               memPr.Seed = expSeeds[rlokal];
    164               memPr.MaximumExecutionTime = TimeSpan.FromSeconds(expPaths);
    165               memPr.StartSync();
    166               if (memPr.Context.RelinkedPaths.IsEmpty) continue;
    167               exp.Add(InstanceDescriptor.FromPaths(qap, memPr.Context.RelinkedPaths.Paths.ToList()));
    168             }
    169             var standardizer = InstancesStandardizer.Create(kb);
    170             standardizer.Apply(kb);
    171             standardizer.Apply(exp);
    172             int exactCount = 0, clsCount = 0, missedCount = 0;
    173             int exactRank = 0, clsRank = 0;
    174             foreach (var e in exp) {
    175               var ordered = kb.OrderBy(x => x.CalculateSimilarity(e)).ToList();
    176               var bestMatch = ordered.First();
    177               if (bestMatch.Cls == e.Cls) {
    178                 clsCount++;
    179                 if (bestMatch.Name == e.Name) exactCount++;
    180               } else missedCount++;
    181               clsRank += ordered.FindIndex((id) => id.Cls == e.Cls) + 1;
    182               exactRank += ordered.FindIndex((id) => id.Name == e.Name) + 1;
    183             }
    184             lock (sync) {
    185               Console.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}\t{5:F2}", rlokal, kbPaths, expPaths, exactCount,
    186                 missedCount, exactRank / (double)exp.Count);
    187             }
    188           }
    189         });
    190       }
     205      } else {
     206        foreach (var point in data) {
     207          var normalizedTest = point.Test.Value.Select(x => new InstanceDescriptor(x)).ToList();
     208          point.Training.Value.Item1.Apply(normalizedTest);
     209          var result = Compare(point.Training.Value.Item2, normalizedTest);
     210          Console.WriteLine("{0}\t{1}\t{2}\t{3:F2}\t{4}\t{5:F2}\t{6}",
     211            point.Training.Key.Effort, point.Test.Key.Effort, result.ExactCount,
     212            result.ExactAverageRank, result.ClsCount, result.ClsAverageRank, result.TotalCount);
     213        }
     214      }
     215    }
     216
     217    private class MatchResult {
     218      public int ExactCount { get; set; }
     219      public int ClsCount { get; set; }
     220      public int TotalCount { get; set; }
     221      public double ExactAverageRank { get; set; }
     222      public double ClsAverageRank { get; set; }
    191223    }
    192224  }
Note: See TracChangeset for help on using the changeset viewer.