Changeset 13787 for branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders

Timestamp:

04/24/16 10:03:52 (9 years ago)

Author:

abeham

Message:

#2457: worked on performance modeling

Location:

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders

Files:

• . (added)
• DistanceWeightedRecommender.cs (moved) (moved from branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/DistanceWeightedRecommender.cs) (4 diffs)
• FixedRankModel.cs (added)
• KNearestNeighborModel.cs (added)
• KNearestNeighborRecommender.cs (moved) (moved from branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/KNearestNeighborRecommender.cs) (4 diffs)
• OverallBestRecommender.cs (moved) (moved from branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/OverallBestRecommender.cs) (5 diffs)

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/DistanceWeightedRecommender.cs

@@ -35 +35 @@
   [StorableClass]
   public class DistanceWeightedRecommender : ParameterizedNamedItem, IAlgorithmInstanceRecommender {
-    private KnowledgeCenter okc;
-
-    private IFixedValueParameter<EnumValue<ProblemInstanceProximityType>> ProximityTypeParameter {
-      get { return (IFixedValueParameter<EnumValue<ProblemInstanceProximityType>>)Parameters["ProximityType"]; }
-    }
 
     private IFixedValueParameter<DoubleValue> NeighborhoodFactorParameter {
       get { return (IFixedValueParameter<DoubleValue>)Parameters["NeighborhoodFactor"]; }
-    }
-
-    public ProblemInstanceProximityType ProximityType {
-      get { return ProximityTypeParameter.Value.Value; }
-      set { ProximityTypeParameter.Value.Value = value; }
     }
 
@@ -59 +49 @@
     private DistanceWeightedRecommender(DistanceWeightedRecommender original, Cloner cloner)
       : base(original, cloner) { }
-    public DistanceWeightedRecommender(KnowledgeCenter okc) {
-      this.okc = okc;
-      Parameters.Add(new FixedValueParameter<EnumValue<ProblemInstanceProximityType>>("ProximityType", "The type of neighbor proximity.", new EnumValue<ProblemInstanceProximityType>(ProblemInstanceProximityType.FeatureSpace)));
+    public DistanceWeightedRecommender() {
       Parameters.Add(new FixedValueParameter<DoubleValue>("NeighborhoodFactor", "Penalize neighbors that are far away.", new DoubleValue(5)));
     }
@@ -69 +57 @@
     }
 
-    public IEnumerable<IAlgorithm> GetRanking() {
-      if (okc.Problem.ProblemId == -1) yield break;
-
-      var piDistances = okc.GetProblemDistances(ProximityType);
+    public IRecommendationModel TrainModel(KnowledgeCenter okc, string[] characteristics) {
+      var piDistances = okc.GetProblemDistances(characteristics);
       var maxDist = piDistances.Max(x => x.Value);
       var instances = new SortedList<double, IAlgorithm>();
@@ -109 +95 @@
           avgERT += new System.Random().NextDouble();
         }
-        instances.Add(avgERT, algorithm);
+        instances.Add(avgERT, (IAlgorithm)algorithm.Clone());
       }
 
-      foreach (var alg in instances.Select(x => (IAlgorithm)x.Value.Clone()))
-        yield return alg;
+      return new FixedRankModel(instances.Select(x => Tuple.Create(x.Value, x.Key)));
     }
   }

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/KNearestNeighborRecommender.cs

@@ -23 +23 @@
 using HeuristicLab.Core;
 using HeuristicLab.Data;
-using HeuristicLab.Optimization;
 using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using System.Collections.Generic;
-using System.Linq;
 
 namespace HeuristicLab.OptimizationExpertSystem.Common {
@@ -33 +30 @@
   [StorableClass]
   public sealed class KNearestNeighborRecommender : ParameterizedNamedItem, IAlgorithmInstanceRecommender {
-    private KnowledgeCenter okc;
-
-    private IFixedValueParameter<EnumValue<ProblemInstanceProximityType>> ProximityTypeParameter {
-      get { return (IFixedValueParameter<EnumValue<ProblemInstanceProximityType>>)Parameters["ProximityType"]; }
-    }
-
-    public ProblemInstanceProximityType ProximityType {
-      get { return ProximityTypeParameter.Value.Value; }
-      set { ProximityTypeParameter.Value.Value = value; }
-    }
-
+    
     private IFixedValueParameter<IntValue> KParameter {
       get { return (IFixedValueParameter<IntValue>)Parameters["K"]; }
@@ -52 +39 @@
     private KNearestNeighborRecommender(KNearestNeighborRecommender original, Cloner cloner)
       : base(original, cloner) { }
-    public KNearestNeighborRecommender(KnowledgeCenter okc) {
-      this.okc = okc;
-      Parameters.Add(new FixedValueParameter<EnumValue<ProblemInstanceProximityType>>("ProximityType", "The type of neighbor proximity.", new EnumValue<ProblemInstanceProximityType>(ProblemInstanceProximityType.FeatureSpace)));
+    public KNearestNeighborRecommender() {
       Parameters.Add(new FixedValueParameter<IntValue>("K", "The number of nearest neighbors to consider.", new IntValue(5)));
     }
@@ -62 +47 @@
     }
 
-    public IEnumerable<IAlgorithm> GetRanking() {
-      if (okc.Problem.ProblemId == -1) yield break;
-
-      var distances = okc.GetProblemDistances(ProximityType);
-      var K = KParameter.Value.Value;
-      var performances = new Dictionary<IAlgorithm, List<double>>();
-      for (var k = 0; k < K; k++) {
-        if (distances.Count == 0) break;
-        var min = distances.MinItems(x => x.Value).First();
-        // lookup algorithm performances in min
-        var perfs = okc.GetAlgorithmPerformance(min.Key);
-        if (perfs.Count == 0) {
-          k--;
-          continue;
-        }
-        foreach (var p in perfs) {
-          var ert = p.Value;
-          if (double.IsNaN(ert)) ert = int.MaxValue;
-          List<double> erts;
-          if (!performances.TryGetValue(p.Key, out erts)) {
-            performances[p.Key] = new List<double>() { ert }; ;
-          } else erts.Add(ert);
-        }
-        distances.Remove(min.Key);
-      }
-      foreach (var alg in performances.Select(x => new { Alg = x.Key, Perf = x.Value.Average() })
-                         .OrderBy(x => x.Perf)
-                         .Select(x => x.Alg))
-        yield return alg;
+    public IRecommendationModel TrainModel(KnowledgeCenter kc, string[] characteristics) {
+      return new KNearestNeighborModel(KParameter.Value.Value, characteristics);
     }
   }

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/OverallBestRecommender.cs

@@ -35 +35 @@
   [StorableClass]
   public class OverallBestRecommender : ParameterizedNamedItem, IAlgorithmInstanceRecommender {
-    private KnowledgeCenter okc;
-
-    private IFixedValueParameter<EnumValue<ProblemInstanceProximityType>> ProximityTypeParameter {
-      get { return (IFixedValueParameter<EnumValue<ProblemInstanceProximityType>>)Parameters["ProximityType"]; }
-    }
 
     private IFixedValueParameter<DoubleValue> NeighborhoodFactorParameter {
       get { return (IFixedValueParameter<DoubleValue>)Parameters["NeighborhoodFactor"]; }
-    }
-
-    public ProblemInstanceProximityType ProximityType {
-      get { return ProximityTypeParameter.Value.Value; }
-      set { ProximityTypeParameter.Value.Value = value; }
     }
 
@@ -59 +49 @@
     private OverallBestRecommender(OverallBestRecommender original, Cloner cloner)
       : base(original, cloner) { }
-    public OverallBestRecommender(KnowledgeCenter okc) {
-      this.okc = okc;
-      Parameters.Add(new FixedValueParameter<EnumValue<ProblemInstanceProximityType>>("ProximityType", "The type of neighbor proximity.", new EnumValue<ProblemInstanceProximityType>(ProblemInstanceProximityType.FeatureSpace)));
+    public OverallBestRecommender() {
       Parameters.Add(new FixedValueParameter<DoubleValue>("NeighborhoodFactor", "Penalize neighbors that are far away.", new DoubleValue(5)));
     }
@@ -69 +57 @@
     }
 
-    public IEnumerable<IAlgorithm> GetRanking() {
-      if (okc.Problem.ProblemId == -1) yield break;
-
+    public IRecommendationModel TrainModel(KnowledgeCenter kc, string[] characteristics) {
       var instances = new List<Tuple<IAlgorithm, double>>();
-      foreach (var relevantRuns in okc.GetKnowledgeBaseByAlgorithm()) {
+      foreach (var relevantRuns in kc.GetKnowledgeBaseByAlgorithm()) {
         var algorithm = relevantRuns.Key;
         var pis = relevantRuns.Value.Select(x => ((StringValue)x.Parameters["Problem Name"]).Value).Distinct()
-                              .Select(x => Tuple.Create(x, okc.ProblemInstances.SingleOrDefault(y => ((StringValue)y.Parameters["Problem Name"]).Value == x)))
+                              .Select(x => Tuple.Create(x, kc.ProblemInstances.SingleOrDefault(y => ((StringValue)y.Parameters["Problem Name"]).Value == x)))
                               .Where(x => x.Item2 != null)
                               .Select(x => Tuple.Create(x.Item1, ((DoubleValue)x.Item2.Parameters["BestKnownQuality"]).Value))
@@ -84 +70 @@
         foreach (var problemRuns in relevantRuns.Value.GroupBy(x => ((StringValue)x.Parameters["Problem Name"]).Value)) {
           var bkq = pis[problemRuns.Key];
-          var ert = ExpectedRuntimeHelper.CalculateErt(problemRuns.ToList(), "QualityPerEvaluations", (okc.Maximization ? (1 - okc.MinimumTarget.Value) : (1 + okc.MinimumTarget.Value)) * bkq, okc.Maximization).ExpectedRuntime;
+          var ert = ExpectedRuntimeHelper.CalculateErt(problemRuns.ToList(), "QualityPerEvaluations", (kc.Maximization ? (1 - kc.MinimumTarget.Value) : (1 + kc.MinimumTarget.Value)) * bkq, kc.Maximization).ExpectedRuntime;
           if (double.IsNaN(ert)) ert = int.MaxValue;
           avgERT += ert;
@@ -90 +76 @@
         }
         avgERT /= count;
-        instances.Add(Tuple.Create(algorithm, avgERT));
+        instances.Add(Tuple.Create((IAlgorithm)algorithm.Clone(), avgERT));
       }
 
-      foreach (var alg in instances.OrderBy(x => x.Item2).Select(x => (IAlgorithm)x.Item1.Clone()))
-        yield return alg;
+      return new FixedRankModel(instances.OrderBy(x => x.Item2));
     }
   }