Free cookie consent management tool by TermsFeed Policy Generator

source: branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/OverallBestRecommender.cs @ 13803

Last change on this file since 13803 was 13803, checked in by abeham, 8 years ago

#2457:

  • changed expected runtime calculation
    • now outputs positive infinity instead of nan when no run was successful
    • now filters outliers in successful and unsuccessful runs by using two standard deviations of the mean of successful runs as lower bound
      • this change allows having unsuccessful runs in the database with low evaluations / runtime (e.g. due to being aborted early or from an experiment where the max budget was lower)
  • worked on recommendation algorithms
    • implemented several performance measures (absolute error, absolute log error, ndcp, kendall's tau) to evaluate the ranking
File size: 3.5 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using HeuristicLab.Analysis;
23using HeuristicLab.Common;
24using HeuristicLab.Core;
25using HeuristicLab.Data;
26using HeuristicLab.Optimization;
27using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
28using System;
29using System.Collections.Generic;
30using System.Linq;
31
32namespace HeuristicLab.OptimizationExpertSystem.Common {
33  [Item("Overall Best Recommender", "")]
34  [StorableClass]
35  public class OverallBestRecommender : ParameterizedNamedItem, IAlgorithmInstanceRecommender {
36
37    private IFixedValueParameter<DoubleValue> NeighborhoodFactorParameter {
38      get { return (IFixedValueParameter<DoubleValue>)Parameters["NeighborhoodFactor"]; }
39    }
40   
41    [StorableConstructor]
42    private OverallBestRecommender(bool deserializing) : base(deserializing) { }
43    private OverallBestRecommender(OverallBestRecommender original, Cloner cloner) : base(original, cloner) { }
44    public OverallBestRecommender() { }
45
46    public override IDeepCloneable Clone(Cloner cloner) {
47      return new OverallBestRecommender(this, cloner);
48    }
49
50    public IRecommendationModel TrainModel(IRun[] problemInstances, KnowledgeCenter kc, string[] characteristics) {
51      var instances = new List<KeyValuePair<IAlgorithm, double>>();
52      foreach (var relevantRuns in kc.GetKnowledgeBaseByAlgorithm()) {
53        var algorithm = relevantRuns.Key;
54        var pis = relevantRuns.Value.Select(x => ((StringValue)x.Parameters["Problem Name"]).Value).Distinct()
55                              .Select(x => Tuple.Create(x, problemInstances.SingleOrDefault(y => ((StringValue)y.Parameters["Problem Name"]).Value == x)))
56                              .Where(x => x.Item2 != null)
57                              .Select(x => Tuple.Create(x.Item1, ((DoubleValue)x.Item2.Parameters["BestKnownQuality"]).Value))
58                              .ToDictionary(x => x.Item1, x => x.Item2);
59        var avgERT = 0.0;
60        var count = 0;
61        foreach (var problemRuns in relevantRuns.Value.GroupBy(x => ((StringValue)x.Parameters["Problem Name"]).Value)) {
62          double bkq;
63          if (!pis.TryGetValue(problemRuns.Key, out bkq)) continue;
64          var ert = ExpectedRuntimeHelper.CalculateErt(problemRuns.ToList(), "QualityPerEvaluations", kc.GetTarget(bkq, kc.MinimumTarget.Value, kc.Maximization), kc.Maximization).ExpectedRuntime;
65          if (double.IsInfinity(ert)) ert = int.MaxValue;
66          avgERT += ert;
67          count++;
68        }
69        avgERT /= count;
70        instances.Add(new KeyValuePair<IAlgorithm, double>(algorithm, avgERT));
71      }
72
73      return new FixedRankModel(instances.OrderBy(x => x.Value));
74    }
75  }
76}
Note: See TracBrowser for help on using the repository browser.