Changeset 13803 for branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/KnowledgeCenter.cs

Timestamp:

04/27/16 16:21:07 (8 years ago)

Author:

abeham

Message:

#2457:

• changed expected runtime calculation
  • now outputs positive infinity instead of nan when no run was successful
  • now filters outliers in successful and unsuccessful runs by using two standard deviations of the mean of successful runs as lower bound
    • this change allows having unsuccessful runs in the database with low evaluations / runtime (e.g. due to being aborted early or from an experiment where the max budget was lower)
• worked on recommendation algorithms
  • implemented several performance measures (absolute error, absolute log error, ndcp, kendall's tau) to evaluate the ranking

File:

• branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/KnowledgeCenter.cs (modified) (2 diffs)

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/KnowledgeCenter.cs

@@ -654 +654 @@
           var bkq = ((DoubleValue)prob.Parameters["BestKnownQuality"]).Value;
           var ert = ExpectedRuntimeHelper.CalculateErt(pr.ToList(), "QualityPerEvaluations", GetTarget(bkq, target, max), max).ExpectedRuntime;
-          if (double.IsNaN(ert)) ert = int.MaxValue;
+          if (double.IsInfinity(ert)) ert = int.MaxValue;
           ds.AddRow(new object[] { pr.Key }.Concat(f.Cast<object>()).Concat(new object[] { ert }));
         }
@@ -720 +720 @@
         var values = pr.GroupBy(x => algorithmId2RunMapping.GetBySecond(x).Single())
                        .ToDictionary(x => x.Key, x => ExpectedRuntimeHelper.CalculateErt(x.ToList(), "QualityPerEvaluations", GetTarget(bkq, target, max), max).ExpectedRuntime);
-        var ranks = ClusteringHelper<long>.Cluster(nClasses, values, x => double.IsNaN(x.Value))
+        var ranks = ClusteringHelper<long>.Cluster(nClasses, values, x => double.IsInfinity(x.Value))
           .GetByCluster().ToList();
         foreach (var c in ranks) {