Changeset 13803 for branches/PerformanceComparison/HeuristicLab.Analysis/3.3/QualityAnalysis/ExpectedRuntimeHelper.cs

Timestamp:

04/27/16 16:21:07 (8 years ago)

Author:

abeham

Message:

#2457:

• changed expected runtime calculation
  • now outputs positive infinity instead of nan when no run was successful
  • now filters outliers in successful and unsuccessful runs by using two standard deviations of the mean of successful runs as lower bound
    • this change allows having unsuccessful runs in the database with low evaluations / runtime (e.g. due to being aborted early or from an experiment where the max budget was lower)
• worked on recommendation algorithms
  • implemented several performance measures (absolute error, absolute log error, ndcp, kendall's tau) to evaluate the ranking

File:

• branches/PerformanceComparison/HeuristicLab.Analysis/3.3/QualityAnalysis/ExpectedRuntimeHelper.cs (modified) (2 diffs)

branches/PerformanceComparison/HeuristicLab.Analysis/3.3/QualityAnalysis/ExpectedRuntimeHelper.cs

@@ -1 @@
-using HeuristicLab.Optimization;
+using HeuristicLab.Common;
+using HeuristicLab.Optimization;
 using System;
 using System.Collections.Generic;
@@ -24 +25 @@
       }
 
-      var ert = double.NaN;
+      var ert = double.PositiveInfinity;
 
+      var nRuns = successful.Count + unsuccessful.Count;
       if (successful.Count > 0) {
-        if (unsuccessful.Count == 0) ert = successful.Average();
-        else {
-          var ps = successful.Count / (double)(successful.Count + unsuccessful.Count);
-          ert = successful.Average() + ((1.0 - ps) / ps) * unsuccessful.Average();
-        }
+        var succAvg = successful.Average();
+        var succDev = successful.StandardDeviation();
+        successful.RemoveAll(x => x < succAvg - 2 * succDev);
+        unsuccessful.RemoveAll(x => x < succAvg - 2 * succDev);
+        nRuns = successful.Count + unsuccessful.Count;
+
+        ert = successful.Average() / (successful.Count / (double)nRuns);
       }
-      return new ErtCalculationResult(successful.Count, (successful.Count + unsuccessful.Count), ert);
+      return new ErtCalculationResult(successful.Count, nRuns, ert);
     }