Changeset 13797 for branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common

Timestamp:

04/26/16 13:07:26 (8 years ago)

Author:

abeham

Message:

#2457: worked on testing recommendation algorithms through x-validation

Location:

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3

Files:

• KnowledgeCenter.cs (modified) (8 diffs)
• Recommenders/OverallBestRecommender.cs (modified) (1 diff)

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/KnowledgeCenter.cs

@@ -421 +421 @@
         var characteristics = new HashSet<string>();
         var totalProblems = adminClient.Problems.Count(x => x.ProblemClassId == probClassId);
-        Parallel.ForEach(adminClient.Problems.Where(x => x.ProblemClassId == probClassId), new ParallelOptions { MaxDegreeOfParallelism = 3 }, (pInst) => { 
+        Parallel.ForEach(adminClient.Problems.Where(x => x.ProblemClassId == probClassId), new ParallelOptions { MaxDegreeOfParallelism = 8 }, (pInst) => { 
           var charas = new List<string>();
           IRun probRun = null;
@@ -457 +457 @@
         progress.ProgressValue = 0;
         p[0] = 0;
-        Parallel.ForEach(adminClient.Algorithms, new ParallelOptions { MaxDegreeOfParallelism = 3 }, (algInst) => {
+        Parallel.ForEach(adminClient.Algorithms, new ParallelOptions { MaxDegreeOfParallelism = 8 }, (algInst) => {
           IAlgorithm alg = null;
           var data = Clients.OKB.Administration.AdministrationClient.GetAlgorithmData(algInst.Id);
@@ -490 +490 @@
         var runIds = queryClient.GetRunIds(problemClassFilter).ToList();
         var batches = runIds.Select((v, i) => new { Idx = i, Val = v }).GroupBy(x => x.Idx / 500, x => x.Val);
-        Parallel.ForEach(batches.Select(x => x.ToList()), new ParallelOptions { MaxDegreeOfParallelism = 3 }, (batch) => {
+        Parallel.ForEach(batches.Select(x => x.ToList()), new ParallelOptions { MaxDegreeOfParallelism = 4 }, (batch) => {
           var okbRuns = queryClient.GetRunsWithValues(batch, true, interestingValues);
           var hlRuns = okbRuns.AsParallel().Select(x => new { AlgorithmId = x.Algorithm.Id, Run = queryClient.ConvertToOptimizationRun(x) }).ToList();
@@ -535 +535 @@
 
         // add algorithm instance ranks as features to the problem instances for a range of targets
-        foreach (var target in new[] {1, 1.01, 1.05, 1.1, 1.2, 1.5}) {
+        foreach (var target in new[] {0, 0.01, 0.05, 0.1, 0.2, 0.5}) {
           var cls = GetPerformanceClasses(target, 5);
           foreach (var kvp in cls) {
             var prob = kvp.Key;
             foreach (var kvp2 in kvp.Value) {
-              var resultName = "Rank." + algorithmId2AlgorithmInstanceMapping.GetByFirst(kvp2.Key) + "@" + ((target - 1) * 100) + "%";
+              var resultName = "Rank." + algorithmId2AlgorithmInstanceMapping.GetByFirst(kvp2.Key) + "@" + (target * 100) + "%";
               prob.Results[resultName] = new IntValue(kvp2.Value);
             }
@@ -617 +617 @@
     public Dictionary<IAlgorithm, double> GetAlgorithmPerformance(IRun problemInstance) {
       if (!problemInstance.Parameters.ContainsKey("BestKnownQuality")) return new Dictionary<IAlgorithm, double>();
-      var target = GetTarget(((DoubleValue)problemInstance.Parameters["BestKnownQuality"]).Value, Maximization);
+      var target = GetTarget(((DoubleValue)problemInstance.Parameters["BestKnownQuality"]).Value, MinimumTarget.Value, Maximization);
       return knowledgeBase.Where(x => ((StringValue)x.Parameters["Problem Name"]).Value == ((StringValue)problemInstance.Parameters["Problem Name"]).Value)
                           .GroupBy(x => algorithmId2AlgorithmInstanceMapping.GetByFirst(algorithmId2RunMapping.GetBySecond(x).Single()))
@@ -653 +653 @@
           var max = ((BoolValue)prob.Parameters["Maximization"]).Value;
           var bkq = ((DoubleValue)prob.Parameters["BestKnownQuality"]).Value;
-          var ert = ExpectedRuntimeHelper.CalculateErt(pr.ToList(), "QualityPerEvaluations", GetTarget(bkq, max), max).ExpectedRuntime;
+          var ert = ExpectedRuntimeHelper.CalculateErt(pr.ToList(), "QualityPerEvaluations", GetTarget(bkq, target, max), max).ExpectedRuntime;
           if (double.IsNaN(ert)) ert = int.MaxValue;
           ds.AddRow(new object[] { pr.Key }.Concat(f.Cast<object>()).Concat(new object[] { ert }));
@@ -719 +719 @@
 
         var values = pr.GroupBy(x => algorithmId2RunMapping.GetBySecond(x).Single())
-                       .ToDictionary(x => x.Key, x => ExpectedRuntimeHelper.CalculateErt(x.ToList(), "QualityPerEvaluations", GetTarget(bkq, max), max).ExpectedRuntime);
+                       .ToDictionary(x => x.Key, x => ExpectedRuntimeHelper.CalculateErt(x.ToList(), "QualityPerEvaluations", GetTarget(bkq, target, max), max).ExpectedRuntime);
         var ranks = ClusteringHelper<long>.Cluster(nClasses, values, x => double.IsNaN(x.Value))
           .GetByCluster().ToList();
-        var minRank = ranks.Min(x => x.Key);
         foreach (var c in ranks) {
           foreach (var a in c.Value)
-            result[problemMap[pr.Key]][a.Key] = c.Key == nClasses ? c.Key : c.Key - minRank;
+            result[problemMap[pr.Key]][a.Key] = c.Key;
         }
       }
@@ -731 +730 @@
     }
 
-    public double GetTarget(double bestKnownQuality, bool maximization) {
-      return bestKnownQuality * (maximization ? (1 - MinimumTarget.Value) : (1 + MinimumTarget.Value));
+    public double GetTarget(double bestKnownQuality, double target, bool maximization) {
+      return bestKnownQuality * (maximization ? (1 - target) : (1 + target));
     }
 

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/OverallBestRecommender.cs

@@ -62 +62 @@
           double bkq;
           if (!pis.TryGetValue(problemRuns.Key, out bkq)) continue;
-          var ert = ExpectedRuntimeHelper.CalculateErt(problemRuns.ToList(), "QualityPerEvaluations", kc.GetTarget(bkq, kc.Maximization), kc.Maximization).ExpectedRuntime;
+          var ert = ExpectedRuntimeHelper.CalculateErt(problemRuns.ToList(), "QualityPerEvaluations", kc.GetTarget(bkq, kc.MinimumTarget.Value, kc.Maximization), kc.Maximization).ExpectedRuntime;
           if (double.IsNaN(ert)) ert = int.MaxValue;
           avgERT += ert;