Changeset 13649 for branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/ExpertSystem.cs

Timestamp:

03/03/16 15:54:48 (8 years ago)

Author:

abeham

Message:

#2457: fixed bugs in suggestions

File:

• branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/ExpertSystem.cs (modified) (8 diffs)

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/ExpertSystem.cs

@@ -20 +20 @@
 #endregion
 
-using System;
-using System.Collections.Generic;
-using System.ComponentModel;
-using System.Drawing;
-using System.IO;
-using System.Linq;
-using System.Threading.Tasks;
 using HeuristicLab.Analysis;
 using HeuristicLab.Collections;
@@ -37 +30 @@
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Persistence.Default.Xml;
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Drawing;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
 using RunCreationClient = HeuristicLab.Clients.OKB.RunCreation.RunCreationClient;
 using SingleObjectiveOKBProblem = HeuristicLab.Clients.OKB.RunCreation.SingleObjectiveOKBProblem;
@@ -108 +109 @@
         problemInstances = value;
         OnPropertyChanged("ProblemInstances");
+      }
+    }
+
+    [Storable]
+    private ResultCollection currentResult;
+    public ResultCollection CurrentResult {
+      get { return currentResult; }
+      set {
+        if (currentResult == value) return;
+        currentResult = value;
+        OnPropertyChanged("CurrentResult");
       }
     }
@@ -327 +339 @@
       "QualityPerEvaluations", "Problem Name", "Problem Type", "Algorithm Name", "Algorithm Type", "Maximization", "BestKnownQuality"
     };
+
+    public async void StartAlgorithmAsync(int index) {
+      var selectedInstance = suggestedInstances[index];
+      var algorithm = (IAlgorithm)selectedInstance.Clone();
+      algorithm.Prepare(true);
+      IParameter stopParam;
+      var monitorStop = true;
+      if (algorithm.Parameters.TryGetValue("MaximumEvaluations", out stopParam)) {
+        var maxEvalParam = stopParam as IValueParameter<Data.IntValue>;
+        if (maxEvalParam != null) {
+          maxEvalParam.Value.Value = MaximumEvaluations;
+          monitorStop = false;
+        }
+      }
+      algorithm.ExecutionStateChanged += AlgorithmOnExecutionStateChanged;
+      algorithm.ExceptionOccurred += AlgorithmOnExceptionOccurred;
+      if (monitorStop) algorithm.ExecutionTimeChanged += AlgorithmOnExecutionTimeChanged;
+
+      algorithm.Start();
+    }
+
+    public void StartAlgorithm(int index) {
+      var selectedInstance = suggestedInstances[index];
+      var algorithm = (IAlgorithm)selectedInstance.Clone();
+      algorithm.Prepare(true);
+      IParameter stopParam;
+      var monitorStop = true;
+      if (algorithm.Parameters.TryGetValue("MaximumEvaluations", out stopParam)) {
+        var maxEvalParam = stopParam as IValueParameter<Data.IntValue>;
+        if (maxEvalParam != null) {
+          maxEvalParam.Value.Value = MaximumEvaluations;
+          monitorStop = false;
+        }
+      }
+      algorithm.ExecutionStateChanged += AlgorithmOnExecutionStateChanged;
+      algorithm.ExceptionOccurred += AlgorithmOnExceptionOccurred;
+      if (monitorStop) algorithm.ExecutionTimeChanged += AlgorithmOnExecutionTimeChanged;
+
+      using (algWh = new AutoResetEvent(false)) {
+        algorithm.Start();
+        algWh.WaitOne();
+      }
+      algWh = null;
+    }
+
+    private AutoResetEvent algWh;
+
+    private void AlgorithmOnExecutionStateChanged(object sender, EventArgs eventArgs) {
+      var alg = sender as IAlgorithm;
+      if (alg == null) return;
+      if (alg.ExecutionState == ExecutionState.Started) {
+        CurrentResult = alg.Results;
+      } else if (alg.ExecutionState == ExecutionState.Stopped) {
+        alg.ExecutionStateChanged -= AlgorithmOnExecutionStateChanged;
+        alg.ExceptionOccurred -= AlgorithmOnExceptionOccurred;
+        alg.ExecutionTimeChanged -= AlgorithmOnExecutionTimeChanged;
+        if (algWh != null) algWh.Set();
+      }
+    }
+
+    private void AlgorithmOnExceptionOccurred(object sender, EventArgs<Exception> eventArgs) {
+      var alg = sender as IAlgorithm;
+      if (alg == null) return;
+      alg.ExecutionStateChanged -= AlgorithmOnExecutionStateChanged;
+      alg.ExceptionOccurred -= AlgorithmOnExceptionOccurred;
+      alg.ExecutionTimeChanged -= AlgorithmOnExecutionTimeChanged;
+      if (algWh != null) algWh.Set();
+    }
+
+    private void AlgorithmOnExecutionTimeChanged(object sender, EventArgs eventArgs) {
+      var alg = sender as IAlgorithm;
+      if (alg == null) return;
+      IResult evalSolResult;
+      if (!alg.Results.TryGetValue("EvaluatedSolutions", out evalSolResult) || !(evalSolResult.Value is Data.IntValue)) return;
+      var evalSols = ((Data.IntValue)evalSolResult.Value).Value;
+      if (evalSols >= MaximumEvaluations) alg.Stop();
+    }
 
     public async void UpdateKnowledgeBaseAsync(IProgress progress) {
@@ -372 +461 @@
                 var probRun = new Run() { Name = prob.Name };
                 prob.CollectParameterValues(probRun.Parameters);
+                probRun.Parameters["Problem Name"] = new StringValue(prob.Name);
+                probRun.Parameters["Problem Type"] = new StringValue(prob.GetType().Name);
                 progress.Status += Environment.NewLine + "Downloading characteristics...";
                 foreach (var v in RunCreationClient.GetCharacteristicValues(problInst.Id)) {
@@ -412 +503 @@
         var count = queryClient.GetNumberOfRuns(problemClassFilter);
         if (count == 0) return;
-
+        
         var runIds = queryClient.GetRunIds(problemClassFilter).ToList();
         var conversions = new List<Task>();
@@ -445 +536 @@
 
           var probInstanceName = ((StringValue)probNameParam).Value;
-          var bkQuality = ((DoubleValue)instance.Parameters["BestKnownQuality"]).Value;
           var maximization = ((BoolValue)instance.Parameters["Maximization"]).Value;
+          
+          IItem bkParam;
+          if (!instance.Parameters.TryGetValue("BestKnownQuality", out bkParam) || !(bkParam is DoubleValue)) {
+            Dictionary<string, List<IRun>> algRuns;
+            if (!algInstRunDict.TryGetValue(probInstanceName, out algRuns)) continue;
+            var list = algRuns.SelectMany(x => x.Value)
+                              .Where(x => x.Results.ContainsKey("QualityPerEvaluations"))
+                              .Select(x => ((IndexedDataTable<double>)x.Results["QualityPerEvaluations"]).Rows.First().Values.Last().Item2);
+            bkParam = new DoubleValue(maximization ? list.Max() : list.Min());
+            instance.Parameters["BestKnownQuality"] = bkParam;
+          } else bkParam = instance.Parameters["BestKnownQuality"];
+
+          var bkQuality = ((DoubleValue)bkParam).Value;
 
           if (!algInstRunDict.ContainsKey(probInstanceName)) continue;
           foreach (var kvp in algInstRunDict[probInstanceName]) {
-            var algInstanceName = kvp.Key;
             // TODO: Things needs to be configurable here (table name, targets)
             foreach (var target in new[] { 1, 1.01, 1.05, 1.1, 1.2, 1.5 }) {
               var result = ExpectedRuntimeHelper.CalculateErt(kvp.Value, "QualityPerEvaluations", bkQuality * target, maximization);
-              var resultName = algInstanceName + "@" + ((target - 1) * 100) + "%";
+              var resultName = kvp.Key + "@" + ((target - 1) * 100) + "%";
               IItem item;
               if (instance.Results.TryGetValue(resultName, out item)) {
@@ -469 +571 @@
       if (Problem == null) return;
       var instances = new SortedList<double, IAlgorithm>();
-      foreach (var relevantRuns in knowledgeBase.GroupBy(x => x.Algorithm)) {
+      foreach (var relevantRuns in knowledgeBase.GroupBy(x => algorithmId2RunMapping.GetBySecond(x).Single())) {
+        var algorithm = algorithmId2AlgorithmInstanceMapping.GetByFirst(relevantRuns.Key);
         var avgQuality = 0.0;
         var counter = 0;
-        foreach (var run in relevantRuns) {
-          var performanceGraph = ((IndexedDataTable<double>)run.Results["QualityPerEvaluations"]);
-          try {
-            avgQuality += performanceGraph.Rows.First().Values.TakeWhile(x => x.Item1 < MaximumEvaluations).Last().Item2;
-            counter++;
-          } catch { continue; }
+        foreach (var problemRuns in relevantRuns.GroupBy(x => ((StringValue)x.Parameters["Problem Name"]).Value)) {
+          var probInstance = ProblemInstances.SingleOrDefault(x => x.Name == problemRuns.Key);
+          if (probInstance == null) continue;
+          var bkQuality = ((DoubleValue)probInstance.Parameters["BestKnownQuality"]).Value;
+          foreach (var run in problemRuns) {
+            var performanceGraph = ((IndexedDataTable<double>)run.Results["QualityPerEvaluations"]);
+            try {
+              avgQuality += performanceGraph.Rows.First().Values.TakeWhile(x => x.Item1 < MaximumEvaluations).Last().Item2 / bkQuality;
+              counter++;
+            } catch {
+              continue;
+            }
+          }
         }
         avgQuality /= counter;
-        instances.Add(avgQuality, relevantRuns.Key);
+        instances.Add(avgQuality, algorithm);
       }
 
       suggestedInstances.Clear();
-      var instanceLadder = instances.Select(x => (IAlgorithm)x.Value.Clone());
-      suggestedInstances.AddRange(Maximization ? instanceLadder.Reverse() : instanceLadder);
+      var instanceLadder = instances.Select(x => (IAlgorithm)x.Value.Clone()).ToList();
+      if (Maximization) instanceLadder.Reverse();
+      suggestedInstances.AddRange(instanceLadder);
     }