Changeset 13649 for branches/PerformanceComparison

Timestamp:

03/03/16 15:54:48 (9 years ago)

Author:

abeham

Message:

#2457: fixed bugs in suggestions

Location:

branches/PerformanceComparison

Files:

• HeuristicLab.OptimizationExpertSystem.Views/3.3/ExpertSystemView.Designer.cs (modified) (1 diff)
• HeuristicLab.OptimizationExpertSystem.Views/3.3/ExpertSystemView.cs (modified) (6 diffs)
• HeuristicLab.OptimizationExpertSystem.Views/3.3/ExpertSystemView.resx (modified) (1 diff)
• HeuristicLab.OptimizationExpertSystem/3.3/ExpertSystem.cs (modified) (8 diffs)

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Views/3.3/ExpertSystemView.Designer.cs

@@ -211 +211 @@
       this.suggestedInstancesComboBox.Size = new System.Drawing.Size(322, 21);
       this.suggestedInstancesComboBox.TabIndex = 0;
+      this.suggestedInstancesComboBox.SelectedIndexChanged += new System.EventHandler(this.SuggestedInstancesComboBoxOnSelectedIndexChanged);
       // 
       // runsTabPage

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Views/3.3/ExpertSystemView.cs

@@ -20 +20 @@
 #endregion
 
+using HeuristicLab.Common.Resources;
+using HeuristicLab.Core.Views;
+using HeuristicLab.MainForm;
+using HeuristicLab.Optimization;
+using HeuristicLab.Optimization.Views;
 using System;
 using System.ComponentModel;
@@ -26 +31 @@
 using System.Windows.Forms;
 using System.Windows.Forms.DataVisualization.Charting;
-using HeuristicLab.Common.Resources;
-using HeuristicLab.Core.Views;
-using HeuristicLab.MainForm;
-using HeuristicLab.Optimization;
-using HeuristicLab.Optimization.Views;
 
 namespace HeuristicLab.OptimizationExpertSystem.Views {
@@ -104 +104 @@
           kbViewHost.Content = Content.KnowledgeBase;
           problemInstancesView.Content = Content.ProblemInstances;
+          algorithmViewHost.Content = Content.CurrentResult;
         }
       } finally { SuppressEvents = false; }
@@ -161 +162 @@
             break;
           case "ProblemInstances": problemInstancesView.Content = Content.ProblemInstances; break;
+          case "CurrentResult": algorithmViewHost.Content = Content.CurrentResult; break;
         }
       } finally { SuppressEvents = false; }
@@ -231 +233 @@
 
     private void AlgorithmStartButtonOnClick(object sender, EventArgs e) {
-      var selectedInstance = (IAlgorithm)suggestedInstancesComboBox.SelectedItem;
-      var clone = (IAlgorithm)selectedInstance.Clone();
-      clone.Prepare(true);
-      clone.Start();
-      algorithmViewHost.Content = clone.Results;
+      if (suggestedInstancesComboBox.SelectedIndex >= 0)
+        Content.StartAlgorithmAsync(suggestedInstancesComboBox.SelectedIndex);
     }
 
@@ -268 +267 @@
       }
     }
+
+    private void SuggestedInstancesComboBoxOnSelectedIndexChanged(object sender, EventArgs e) {
+      SetEnabledStateOfControls();
+    }
   }
 }

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Views/3.3/ExpertSystemView.resx

@@ -121 +121 @@
     <value>17, 17</value>
   </metadata>
-  <metadata name="errorProvider.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
-    <value>17, 17</value>
-  </metadata>
   <metadata name="toolTip.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
     <value>140, 17</value>

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/ExpertSystem.cs

@@ -20 +20 @@
 #endregion
 
-using System;
-using System.Collections.Generic;
-using System.ComponentModel;
-using System.Drawing;
-using System.IO;
-using System.Linq;
-using System.Threading.Tasks;
 using HeuristicLab.Analysis;
 using HeuristicLab.Collections;
@@ -37 +30 @@
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Persistence.Default.Xml;
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Drawing;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
 using RunCreationClient = HeuristicLab.Clients.OKB.RunCreation.RunCreationClient;
 using SingleObjectiveOKBProblem = HeuristicLab.Clients.OKB.RunCreation.SingleObjectiveOKBProblem;
@@ -108 +109 @@
         problemInstances = value;
         OnPropertyChanged("ProblemInstances");
+      }
+    }
+
+    [Storable]
+    private ResultCollection currentResult;
+    public ResultCollection CurrentResult {
+      get { return currentResult; }
+      set {
+        if (currentResult == value) return;
+        currentResult = value;
+        OnPropertyChanged("CurrentResult");
       }
     }
@@ -327 +339 @@
       "QualityPerEvaluations", "Problem Name", "Problem Type", "Algorithm Name", "Algorithm Type", "Maximization", "BestKnownQuality"
     };
+
+    public async void StartAlgorithmAsync(int index) {
+      var selectedInstance = suggestedInstances[index];
+      var algorithm = (IAlgorithm)selectedInstance.Clone();
+      algorithm.Prepare(true);
+      IParameter stopParam;
+      var monitorStop = true;
+      if (algorithm.Parameters.TryGetValue("MaximumEvaluations", out stopParam)) {
+        var maxEvalParam = stopParam as IValueParameter<Data.IntValue>;
+        if (maxEvalParam != null) {
+          maxEvalParam.Value.Value = MaximumEvaluations;
+          monitorStop = false;
+        }
+      }
+      algorithm.ExecutionStateChanged += AlgorithmOnExecutionStateChanged;
+      algorithm.ExceptionOccurred += AlgorithmOnExceptionOccurred;
+      if (monitorStop) algorithm.ExecutionTimeChanged += AlgorithmOnExecutionTimeChanged;
+
+      algorithm.Start();
+    }
+
+    public void StartAlgorithm(int index) {
+      var selectedInstance = suggestedInstances[index];
+      var algorithm = (IAlgorithm)selectedInstance.Clone();
+      algorithm.Prepare(true);
+      IParameter stopParam;
+      var monitorStop = true;
+      if (algorithm.Parameters.TryGetValue("MaximumEvaluations", out stopParam)) {
+        var maxEvalParam = stopParam as IValueParameter<Data.IntValue>;
+        if (maxEvalParam != null) {
+          maxEvalParam.Value.Value = MaximumEvaluations;
+          monitorStop = false;
+        }
+      }
+      algorithm.ExecutionStateChanged += AlgorithmOnExecutionStateChanged;
+      algorithm.ExceptionOccurred += AlgorithmOnExceptionOccurred;
+      if (monitorStop) algorithm.ExecutionTimeChanged += AlgorithmOnExecutionTimeChanged;
+
+      using (algWh = new AutoResetEvent(false)) {
+        algorithm.Start();
+        algWh.WaitOne();
+      }
+      algWh = null;
+    }
+
+    private AutoResetEvent algWh;
+
+    private void AlgorithmOnExecutionStateChanged(object sender, EventArgs eventArgs) {
+      var alg = sender as IAlgorithm;
+      if (alg == null) return;
+      if (alg.ExecutionState == ExecutionState.Started) {
+        CurrentResult = alg.Results;
+      } else if (alg.ExecutionState == ExecutionState.Stopped) {
+        alg.ExecutionStateChanged -= AlgorithmOnExecutionStateChanged;
+        alg.ExceptionOccurred -= AlgorithmOnExceptionOccurred;
+        alg.ExecutionTimeChanged -= AlgorithmOnExecutionTimeChanged;
+        if (algWh != null) algWh.Set();
+      }
+    }
+
+    private void AlgorithmOnExceptionOccurred(object sender, EventArgs<Exception> eventArgs) {
+      var alg = sender as IAlgorithm;
+      if (alg == null) return;
+      alg.ExecutionStateChanged -= AlgorithmOnExecutionStateChanged;
+      alg.ExceptionOccurred -= AlgorithmOnExceptionOccurred;
+      alg.ExecutionTimeChanged -= AlgorithmOnExecutionTimeChanged;
+      if (algWh != null) algWh.Set();
+    }
+
+    private void AlgorithmOnExecutionTimeChanged(object sender, EventArgs eventArgs) {
+      var alg = sender as IAlgorithm;
+      if (alg == null) return;
+      IResult evalSolResult;
+      if (!alg.Results.TryGetValue("EvaluatedSolutions", out evalSolResult) || !(evalSolResult.Value is Data.IntValue)) return;
+      var evalSols = ((Data.IntValue)evalSolResult.Value).Value;
+      if (evalSols >= MaximumEvaluations) alg.Stop();
+    }
 
     public async void UpdateKnowledgeBaseAsync(IProgress progress) {
@@ -372 +461 @@
                 var probRun = new Run() { Name = prob.Name };
                 prob.CollectParameterValues(probRun.Parameters);
+                probRun.Parameters["Problem Name"] = new StringValue(prob.Name);
+                probRun.Parameters["Problem Type"] = new StringValue(prob.GetType().Name);
                 progress.Status += Environment.NewLine + "Downloading characteristics...";
                 foreach (var v in RunCreationClient.GetCharacteristicValues(problInst.Id)) {
@@ -412 +503 @@
         var count = queryClient.GetNumberOfRuns(problemClassFilter);
         if (count == 0) return;
-
+        
         var runIds = queryClient.GetRunIds(problemClassFilter).ToList();
         var conversions = new List<Task>();
@@ -445 +536 @@
 
           var probInstanceName = ((StringValue)probNameParam).Value;
-          var bkQuality = ((DoubleValue)instance.Parameters["BestKnownQuality"]).Value;
           var maximization = ((BoolValue)instance.Parameters["Maximization"]).Value;
+          
+          IItem bkParam;
+          if (!instance.Parameters.TryGetValue("BestKnownQuality", out bkParam) || !(bkParam is DoubleValue)) {
+            Dictionary<string, List<IRun>> algRuns;
+            if (!algInstRunDict.TryGetValue(probInstanceName, out algRuns)) continue;
+            var list = algRuns.SelectMany(x => x.Value)
+                              .Where(x => x.Results.ContainsKey("QualityPerEvaluations"))
+                              .Select(x => ((IndexedDataTable<double>)x.Results["QualityPerEvaluations"]).Rows.First().Values.Last().Item2);
+            bkParam = new DoubleValue(maximization ? list.Max() : list.Min());
+            instance.Parameters["BestKnownQuality"] = bkParam;
+          } else bkParam = instance.Parameters["BestKnownQuality"];
+
+          var bkQuality = ((DoubleValue)bkParam).Value;
 
           if (!algInstRunDict.ContainsKey(probInstanceName)) continue;
           foreach (var kvp in algInstRunDict[probInstanceName]) {
-            var algInstanceName = kvp.Key;
             // TODO: Things needs to be configurable here (table name, targets)
             foreach (var target in new[] { 1, 1.01, 1.05, 1.1, 1.2, 1.5 }) {
               var result = ExpectedRuntimeHelper.CalculateErt(kvp.Value, "QualityPerEvaluations", bkQuality * target, maximization);
-              var resultName = algInstanceName + "@" + ((target - 1) * 100) + "%";
+              var resultName = kvp.Key + "@" + ((target - 1) * 100) + "%";
               IItem item;
               if (instance.Results.TryGetValue(resultName, out item)) {
@@ -469 +571 @@
       if (Problem == null) return;
       var instances = new SortedList<double, IAlgorithm>();
-      foreach (var relevantRuns in knowledgeBase.GroupBy(x => x.Algorithm)) {
+      foreach (var relevantRuns in knowledgeBase.GroupBy(x => algorithmId2RunMapping.GetBySecond(x).Single())) {
+        var algorithm = algorithmId2AlgorithmInstanceMapping.GetByFirst(relevantRuns.Key);
         var avgQuality = 0.0;
         var counter = 0;
-        foreach (var run in relevantRuns) {
-          var performanceGraph = ((IndexedDataTable<double>)run.Results["QualityPerEvaluations"]);
-          try {
-            avgQuality += performanceGraph.Rows.First().Values.TakeWhile(x => x.Item1 < MaximumEvaluations).Last().Item2;
-            counter++;
-          } catch { continue; }
+        foreach (var problemRuns in relevantRuns.GroupBy(x => ((StringValue)x.Parameters["Problem Name"]).Value)) {
+          var probInstance = ProblemInstances.SingleOrDefault(x => x.Name == problemRuns.Key);
+          if (probInstance == null) continue;
+          var bkQuality = ((DoubleValue)probInstance.Parameters["BestKnownQuality"]).Value;
+          foreach (var run in problemRuns) {
+            var performanceGraph = ((IndexedDataTable<double>)run.Results["QualityPerEvaluations"]);
+            try {
+              avgQuality += performanceGraph.Rows.First().Values.TakeWhile(x => x.Item1 < MaximumEvaluations).Last().Item2 / bkQuality;
+              counter++;
+            } catch {
+              continue;
+            }
+          }
         }
         avgQuality /= counter;
-        instances.Add(avgQuality, relevantRuns.Key);
+        instances.Add(avgQuality, algorithm);
       }
 
       suggestedInstances.Clear();
-      var instanceLadder = instances.Select(x => (IAlgorithm)x.Value.Clone());
-      suggestedInstances.AddRange(Maximization ? instanceLadder.Reverse() : instanceLadder);
+      var instanceLadder = instances.Select(x => (IAlgorithm)x.Value.Clone()).ToList();
+      if (Maximization) instanceLadder.Reverse();
+      suggestedInstances.AddRange(instanceLadder);
     }