Changeset 13791

Timestamp:

04/25/16 12:56:17 (8 years ago)

Author:

abeham

Message:

#2457: working on recommendation algorithms

Location:

branches/PerformanceComparison

Files:

• HeuristicLab.OptimizationExpertSystem.Common/3.3/HeuristicLab.OptimizationExpertSystem.Common-3.3.csproj (modified) (1 diff)
• HeuristicLab.OptimizationExpertSystem.Common/3.3/Interfaces/IAlgorithmInstanceRecommender.cs (modified) (1 diff)
• HeuristicLab.OptimizationExpertSystem.Common/3.3/Interfaces/IRecommendationModel.cs (modified) (1 diff)
• HeuristicLab.OptimizationExpertSystem.Common/3.3/KnowledgeCenter.cs (modified) (8 diffs)
• HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/FixedRankModel.cs (modified) (2 diffs)
• HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/KNearestNeighborModel.cs (modified) (3 diffs)
• HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/KNearestNeighborRecommender.cs (modified) (3 diffs)
• HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/OverallBestRecommender.cs (modified) (4 diffs)
• HeuristicLab.OptimizationExpertSystem/3.3/Menu/100_Understanding/110_ProblemInstanceMenuItem.cs (modified) (1 diff)
• HeuristicLab.OptimizationExpertSystem/3.3/Menu/200_Solving/210_PerformanceModelingMenuItem.cs (modified) (1 diff)
• HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.Designer.cs (modified) (8 diffs)
• HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.cs (modified) (6 diffs)
• HeuristicLab.OptimizationExpertSystem/3.3/Views/SolverView.Designer.cs (modified) (16 diffs)
• HeuristicLab.OptimizationExpertSystem/3.3/Views/SolverView.cs (modified) (3 diffs)
• HeuristicLab.OptimizationExpertSystem/3.3/Views/UnderstandingProblemInstanceView.Designer.cs (modified) (12 diffs)
• HeuristicLab.OptimizationExpertSystem/3.3/Views/UnderstandingProblemInstanceView.cs (modified) (11 diffs)

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/HeuristicLab.OptimizationExpertSystem.Common-3.3.csproj

@@ -161 +161 @@
     <Compile Include="Recommenders\KNearestNeighborModel.cs" />
     <Compile Include="Recommenders\OverallBestRecommender.cs" />
-    <Compile Include="Recommenders\DistanceWeightedRecommender.cs" />
     <Compile Include="Interfaces\IAlgorithmInstanceRecommender.cs" />
     <Compile Include="Recommenders\KNearestNeighborRecommender.cs" />

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Interfaces/IAlgorithmInstanceRecommender.cs

@@ -21 +21 @@
 
 using HeuristicLab.Core;
+using HeuristicLab.Optimization;
 
 namespace HeuristicLab.OptimizationExpertSystem.Common {
   public interface IAlgorithmInstanceRecommender : IParameterizedItem {
-    IRecommendationModel TrainModel(KnowledgeCenter kc, string[] characteristics);
+    IRecommendationModel TrainModel(IRun[] problemInstances, KnowledgeCenter kc, string[] characteristics);
   }
 }

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Interfaces/IRecommendationModel.cs

@@ -27 +27 @@
 namespace HeuristicLab.OptimizationExpertSystem.Common {
   public interface IRecommendationModel : IContent {
-    IEnumerable<Tuple<IAlgorithm, double>> GetRanking(KnowledgeCenter kc);
+    IEnumerable<Tuple<IAlgorithm, double>> GetRanking(IRun problemInstance);
   }
 }

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/KnowledgeCenter.cs

@@ -212 +212 @@
       #endregion
       #region PCA
+      double[,] v = null;
       var ds = new double[instances.Count, characteristics.Length];
-      foreach (var instance in instances) {
-        var arr = instance.Value;
-        for (var feature = 0; feature < arr.Length; feature++)
-          ds[key2Idx.GetByFirst(instance.Key), feature] = arr[feature];
-      }
-
-      int info;
-      double[] s2;
-      double[,] v;
-      alglib.pcabuildbasis(ds, ds.GetLength(0), ds.GetLength(1), out info, out s2, out v);
+      if (characteristics.Length > 1) {
+        foreach (var instance in instances) {
+          var arr = instance.Value;
+          for (var feature = 0; feature < arr.Length; feature++)
+            ds[key2Idx.GetByFirst(instance.Key), feature] = arr[feature];
+        }
+
+        int info;
+        double[] s2;
+        alglib.pcabuildbasis(ds, ds.GetLength(0), ds.GetLength(1), out info, out s2, out v);
+      }
       #endregion
       #region SOM
@@ -231 +233 @@
           features[feature, key2Idx.GetByFirst(instance.Key)] = arr[feature];
       }
-      var somCoords = SOM.Map(features, new MersenneTwister(42), somSize: 20, learningRadius: 20, iterations: 200, jittering: true);
+      var somCoords = SOM.Map(features, new MersenneTwister(42), somSize: 10, learningRadius: 20, iterations: 200, jittering: true);
       #endregion
 
@@ -237 +239 @@
       try {
         foreach (var instance in ProblemInstances) {
-          double x = 0, y = 0;
-          for (var feature = 0; feature < ds.GetLength(1); feature++) {
-            x += ds[key2Idx.GetByFirst(instance), feature] * v[feature, 0];
-            y += ds[key2Idx.GetByFirst(instance), feature] * v[feature, 1];
-          }
-
           IItem item;
-          if (instance.Results.TryGetValue("Projection.PCA.X", out item)) {
-            ((DoubleValue)item).Value = x;
-          } else instance.Results.Add("Projection.PCA.X", new DoubleValue(x));
-          if (instance.Results.TryGetValue("Projection.PCA.Y", out item)) {
-            ((DoubleValue)item).Value = y;
-          } else instance.Results.Add("Projection.PCA.Y", new DoubleValue(y));
+          if (v != null) {
+            double x = 0, y = 0;
+            for (var feature = 0; feature < ds.GetLength(1); feature++) {
+              x += ds[key2Idx.GetByFirst(instance), feature] * v[feature, 0];
+              y += ds[key2Idx.GetByFirst(instance), feature] * v[feature, 1];
+            }
+
+            if (instance.Results.TryGetValue("Projection.PCA.X", out item)) {
+              ((DoubleValue)item).Value = x;
+            } else instance.Results.Add("Projection.PCA.X", new DoubleValue(x));
+            if (instance.Results.TryGetValue("Projection.PCA.Y", out item)) {
+              ((DoubleValue)item).Value = y;
+            } else instance.Results.Add("Projection.PCA.Y", new DoubleValue(y));
+          } else {
+            instance.Results.Remove("Projection.PCA.X");
+            instance.Results.Remove("Projection.PCA.Y");
+          }
 
           if (instance.Results.TryGetValue("Projection.MDS.X", out item)) {
@@ -266 +273 @@
         }
       } finally { ProblemInstances.UpdateOfRunsInProgress = false; }
-    }
-
-    public Dictionary<IRun, double[]> GetProblemCharacteristics(string[] characteristics) {
-      var instances = new Dictionary<IRun, double[]>();
-      var values = new List<double>[characteristics.Length];
-      foreach (var run in ProblemInstances) {
-        var f = 0;
-        instances[run] = new double[characteristics.Length];
-        foreach (var c in characteristics) {
-          if (values[f] == null) values[f] = new List<double>(ProblemInstances.Count);
-          IItem item;
-          if (run.Results.TryGetValue(c, out item)) {
-            var val = (double)((dynamic)item).Value;
-            if (!double.IsNaN(val)) values[f].Add(val);
-            instances[run][f] = val;
-          } else instances[run][f] = double.NaN;
-          f++;
-        }
-      }
-      var median = values.Select(x => x.Count == 0 ? 0.0 : x.Median()).ToList();
-
-      var allValues = instances.Values.Select(x => x.Select((f, i) => new {Idx = i, Val = double.IsNaN(f) ? median[i] : f}).ToList())
-        .SelectMany(x => x)
-        .GroupBy(x => x.Idx, x => x.Val)
-        .OrderBy(x => x.Key).ToList();
-      var avg = allValues.Select(x => x.Average()).ToList();
-      var stdev = allValues.Select(x => x.StandardDeviation()).ToList();
-
-      // normalize characteristic values by transforming them to their z-score
-      foreach (var key in instances.Keys.ToList()) {
-        var arr = instances[key];
-        for (var i = 0; i < arr.Length; i++) {
-          if (double.IsNaN(arr[i])) arr[i] = median[i];
-          if (stdev[i] > 0) arr[i] = (arr[i] - avg[i]) / stdev[i];
-        }
-      }
-      return instances;
     }
 
@@ -579 +549 @@
     }
 
+    public static double[][] GetFeatures(IRun[] problemInstances, string[] characteristics, double[] medianValues = null) {
+      var instances = new double[problemInstances.Length][];
+      for (var p = 0; p < problemInstances.Length; p++) {
+        instances[p] = new double[characteristics.Length];
+        for (var f = 0; f < characteristics.Length; f++) {
+          IItem item;
+          if (problemInstances[p].Results.TryGetValue(characteristics[f], out item)) {
+            double val = 0;
+            var dItem = item as DoubleValue;
+            if (dItem != null) {
+              val = dItem.Value;
+            } else {
+              var iItem = item as IntValue;
+              if (iItem != null) val = iItem.Value;
+              else val = double.NaN;
+            }
+            if (double.IsNaN(val) && medianValues != null)
+              instances[p][f] = medianValues[f];
+            else instances[p][f] = val;
+          } else instances[p][f] = medianValues != null ? medianValues[f] : double.NaN;
+        }
+      }
+      return instances;
+    }
+
+    public static double[] GetMedianValues(IRun[] problemInstances, string[] characteristics) {
+      var values = new List<double>[characteristics.Length];
+      foreach (var problemInstance in problemInstances) {
+        for (var f = 0; f < characteristics.Length; f++) {
+          if (values[f] == null) values[f] = new List<double>(problemInstances.Length);
+          IItem item;
+          if (problemInstance.Results.TryGetValue(characteristics[f], out item)) {
+            var dItem = item as DoubleValue;
+            if (dItem != null) values[f].Add(dItem.Value);
+            else {
+              var iItem = item as IntValue;
+              if (iItem != null) values[f].Add(iItem.Value);
+            }
+          }
+        }
+      }
+      return values.Select(x => x.Count == 0 ? 0.0 : x.Median()).ToArray();
+    }
+
+    public Dictionary<IRun, double[]> GetProblemCharacteristics(string[] characteristics) {
+      var map = ProblemInstances.Select((v, i) => new { Index = i, ProblemInstance = v }).ToDictionary(x => x.Index, x => x.ProblemInstance);
+      var instances = GetFeatures(ProblemInstances.ToArray(), characteristics);
+      var median = GetMedianValues(ProblemInstances.ToArray(), characteristics);
+
+      var allValues = instances.Select(x => x.Select((f, i) => new { Idx = i, Val = double.IsNaN(f) ? median[i] : f }).ToList())
+        .SelectMany(x => x)
+        .GroupBy(x => x.Idx, x => x.Val)
+        .OrderBy(x => x.Key).ToList();
+      var avg = allValues.Select(x => x.Average()).ToList();
+      var stdev = allValues.Select(x => x.StandardDeviation()).ToList();
+
+      // normalize characteristic values by transforming them to their z-score
+      foreach (var features in instances) {
+        for (var i = 0; i < features.Length; i++) {
+          if (double.IsNaN(features[i])) features[i] = median[i];
+          if (stdev[i] > 0) features[i] = (features[i] - avg[i]) / stdev[i];
+        }
+      }
+      return instances.Select((v, i) => new { ProblemInstance = map[i], Features = v }).ToDictionary(x => x.ProblemInstance, x => x.Features);
+    }
+
     public Dictionary<IAlgorithm, double> GetAlgorithmPerformance(IRun problemInstance) {
       if (!problemInstance.Parameters.ContainsKey("BestKnownQuality")) return new Dictionary<IAlgorithm, double>();
@@ -586 +622 @@
                           .ToDictionary(x => x.Key, x => ExpectedRuntimeHelper.CalculateErt(x.ToList(), "QualityPerEvaluations", target, Maximization).ExpectedRuntime);
     }
+
+    public Dictionary<IAlgorithm, List<IRun>> GetAlgorithmRuns(IRun problemInstance) {
+      return knowledgeBase.Where(x => ((StringValue)x.Parameters["Problem Name"]).Value == ((StringValue)problemInstance.Parameters["Problem Name"]).Value)
+                          .GroupBy(x => algorithmId2AlgorithmInstanceMapping.GetByFirst(algorithmId2RunMapping.GetBySecond(x).Single()))
+                          .ToDictionary(x => x.Key, x => x.ToList());
+    } 
 
     public Dictionary<IAlgorithm, List<IRun>> GetKnowledgeBaseByAlgorithm() {
@@ -708 +750 @@
     }
 
-    private double GetTarget(double bestKnownQuality, bool maximization) {
+    public double GetTarget(double bestKnownQuality, bool maximization) {
       return bestKnownQuality * (maximization ? (1 - MinimumTarget.Value) : (1 + MinimumTarget.Value));
     }
@@ -817 +859 @@
 
     public IEnumerable<Tuple<IAlgorithm, double>> GetAlgorithmInstanceRanking() {
-      return RecommendationModel.GetRanking(this);
+      return RecommendationModel.GetRanking(ProblemInstances.Single(IsCurrentInstance));
     }
   }

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/FixedRankModel.cs

@@ -26 +26 @@
 namespace HeuristicLab.OptimizationExpertSystem.Common {
   public class FixedRankModel : IRecommendationModel {
-    private List<Tuple<IAlgorithm, double>> ranking;
+    private readonly List<Tuple<IAlgorithm, double>> ranking;
 
     public FixedRankModel(IEnumerable<Tuple<IAlgorithm, double>> ranking) {
@@ -33 +33 @@
     }
  
-    public IEnumerable<Tuple<IAlgorithm, double>> GetRanking(KnowledgeCenter kc) { 
-      if (kc.Problem.ProblemId == -1) return new Tuple<IAlgorithm, double>[0];
+    public IEnumerable<Tuple<IAlgorithm, double>> GetRanking(IRun problemInstance) {
       return ranking;
     }

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/KNearestNeighborModel.cs

@@ -20 +20 @@
 #endregion
 
-using HeuristicLab.Common;
+using HeuristicLab.Collections;
 using HeuristicLab.Optimization;
 using System;
@@ -30 +30 @@
     private readonly int K;
     private readonly string[] characteristics;
+    private readonly Dictionary<IRun, Dictionary<IAlgorithm, double>> performance;
+    private readonly BidirectionalDictionary<int, IRun> problemInstanceMap; 
+    private readonly double[] medianValues;
 
-    public KNearestNeighborModel(int k, string[] characteristics) {
+    public KNearestNeighborModel(int k, Dictionary<IRun, Dictionary<IAlgorithm, double>> perfData, string[] characteristics) {
       this.K = k;
+      this.performance = perfData;
       this.characteristics = characteristics;
+      problemInstanceMap = new BidirectionalDictionary<int, IRun>();
+      var i = 0;
+      foreach (var pi in perfData.Keys) {
+        problemInstanceMap.Add(i++, pi);
+      }
+      this.medianValues = KnowledgeCenter.GetMedianValues(perfData.Keys.OrderBy(problemInstanceMap.GetBySecond).ToArray(), characteristics);
     }
 
-    public IEnumerable<Tuple<IAlgorithm, double>> GetRanking(KnowledgeCenter kc) {
-      var distances = kc.GetProblemDistances(characteristics);
+    public IEnumerable<Tuple<IAlgorithm, double>> GetRanking(IRun problemInstance) {
+      var features = KnowledgeCenter.GetFeatures(performance.Keys.OrderBy(problemInstanceMap.GetBySecond).ToArray(), characteristics, medianValues);
+      var feature = KnowledgeCenter.GetFeatures(new [] { problemInstance }, characteristics, medianValues)[0];
+      var nearestK = features.Select((f, i) => new { ProblemInstanceIndex = i, Feature = f })
+                             .OrderBy(x => x.Feature.Select((f, i) => (f - feature[i]) * (f - feature[i])).Sum())
+                             .Take(K);
+      
       var performances = new Dictionary<IAlgorithm, List<double>>();
-      for (var k = 0; k < K; k++) {
-        if (distances.Count == 0) break;
-        var min = distances.MinItems(x => x.Value).First();
-        // lookup algorithm performances in min
-        var perfs = kc.GetAlgorithmPerformance(min.Key);
-        if (perfs.Count == 0) {
-          k--;
-          continue;
-        }
+      foreach (var next in nearestK) {
+        var perfs = performance[problemInstanceMap.GetByFirst(next.ProblemInstanceIndex)];
+        if (perfs.Count == 0) continue;
+        
         foreach (var p in perfs) {
           var ert = p.Value;
@@ -56 +66 @@
           } else erts.Add(ert);
         }
-        distances.Remove(min.Key);
       }
 

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/KNearestNeighborRecommender.cs

@@ -23 +23 @@
 using HeuristicLab.Core;
 using HeuristicLab.Data;
+using HeuristicLab.Optimization;
 using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using System.Linq;
 
 namespace HeuristicLab.OptimizationExpertSystem.Common {
@@ -37 +39 @@
     [StorableConstructor]
     private KNearestNeighborRecommender(bool deserializing) : base(deserializing) { }
-    private KNearestNeighborRecommender(KNearestNeighborRecommender original, Cloner cloner)
-      : base(original, cloner) { }
+    private KNearestNeighborRecommender(KNearestNeighborRecommender original, Cloner cloner) : base(original, cloner) { }
     public KNearestNeighborRecommender() {
-      Parameters.Add(new FixedValueParameter<IntValue>("K", "The number of nearest neighbors to consider.", new IntValue(5)));
+      Parameters.Add(new FixedValueParameter<IntValue>("K", "The number of nearest neighbors to consider.", new IntValue(3)));
     }
 
@@ -47 +48 @@
     }
 
-    public IRecommendationModel TrainModel(KnowledgeCenter kc, string[] characteristics) {
-      return new KNearestNeighborModel(KParameter.Value.Value, characteristics);
+    public IRecommendationModel TrainModel(IRun[] problemInstances, KnowledgeCenter kc, string[] characteristics) {
+      var perfData = problemInstances.Select(pi => new { ProblemInstance = pi, Performance = kc.GetAlgorithmPerformance(pi) })
+                                     .ToDictionary(x => x.ProblemInstance, x => x.Performance);
+      return new KNearestNeighborModel(KParameter.Value.Value, perfData, characteristics);
     }
   }

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/OverallBestRecommender.cs

@@ -25 +25 @@
 using HeuristicLab.Data;
 using HeuristicLab.Optimization;
-using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using System;
@@ -39 +38 @@
       get { return (IFixedValueParameter<DoubleValue>)Parameters["NeighborhoodFactor"]; }
     }
-
-    public double NeighborhoodFactor {
-      get { return NeighborhoodFactorParameter.Value.Value; }
-      set { NeighborhoodFactorParameter.Value.Value = value; }
-    }
     
     [StorableConstructor]
     private OverallBestRecommender(bool deserializing) : base(deserializing) { }
-    private OverallBestRecommender(OverallBestRecommender original, Cloner cloner)
-      : base(original, cloner) { }
-    public OverallBestRecommender() {
-      Parameters.Add(new FixedValueParameter<DoubleValue>("NeighborhoodFactor", "Penalize neighbors that are far away.", new DoubleValue(5)));
-    }
+    private OverallBestRecommender(OverallBestRecommender original, Cloner cloner) : base(original, cloner) { }
+    public OverallBestRecommender() { }
 
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -57 +48 @@
     }
 
-    public IRecommendationModel TrainModel(KnowledgeCenter kc, string[] characteristics) {
+    public IRecommendationModel TrainModel(IRun[] problemInstances, KnowledgeCenter kc, string[] characteristics) {
       var instances = new List<Tuple<IAlgorithm, double>>();
       foreach (var relevantRuns in kc.GetKnowledgeBaseByAlgorithm()) {
         var algorithm = relevantRuns.Key;
         var pis = relevantRuns.Value.Select(x => ((StringValue)x.Parameters["Problem Name"]).Value).Distinct()
-                              .Select(x => Tuple.Create(x, kc.ProblemInstances.SingleOrDefault(y => ((StringValue)y.Parameters["Problem Name"]).Value == x)))
+                              .Select(x => Tuple.Create(x, problemInstances.SingleOrDefault(y => ((StringValue)y.Parameters["Problem Name"]).Value == x)))
                               .Where(x => x.Item2 != null)
                               .Select(x => Tuple.Create(x.Item1, ((DoubleValue)x.Item2.Parameters["BestKnownQuality"]).Value))
@@ -70 +61 @@
         foreach (var problemRuns in relevantRuns.Value.GroupBy(x => ((StringValue)x.Parameters["Problem Name"]).Value)) {
           var bkq = pis[problemRuns.Key];
-          var ert = ExpectedRuntimeHelper.CalculateErt(problemRuns.ToList(), "QualityPerEvaluations", (kc.Maximization ? (1 - kc.MinimumTarget.Value) : (1 + kc.MinimumTarget.Value)) * bkq, kc.Maximization).ExpectedRuntime;
+          var ert = ExpectedRuntimeHelper.CalculateErt(problemRuns.ToList(), "QualityPerEvaluations", kc.GetTarget(bkq, kc.Maximization), kc.Maximization).ExpectedRuntime;
           if (double.IsNaN(ert)) ert = int.MaxValue;
           avgERT += ert;

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Menu/100_Understanding/110_ProblemInstanceMenuItem.cs

@@ -28 +28 @@
 namespace HeuristicLab.OptimizationExpertSystem.Menu {
   internal class ProblemInstanceMenuItem : MenuItemBase {
-    public override Image Image { get { return VSImageLibrary.Statistics; } }
+    public override Image Image { get { return VSImageLibrary.RadialChart; } }
 
     public override string Name {

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Menu/200_Solving/210_PerformanceModelingMenuItem.cs

@@ -28 +28 @@
 namespace HeuristicLab.OptimizationExpertSystem.Menu {
   internal class PerformanceModelingMenuItem : MenuItemBase {
-    public override Image Image { get { return VSImageLibrary.Event; } }
+    public override Image Image { get { return VSImageLibrary.Graph; } }
 
     public override string Name {

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.Designer.cs

@@ -50 +50 @@
       this.tabControl = new System.Windows.Forms.TabControl();
       this.characteristicsTabPage = new System.Windows.Forms.TabPage();
+      this.characteristicsViewHost = new HeuristicLab.MainForm.WindowsForms.ViewHost();
       this.refreshCharacteristicsButton = new System.Windows.Forms.Button();
       this.parametersTabPage = new System.Windows.Forms.TabPage();
       this.parameterCollectionView = new HeuristicLab.Core.Views.ParameterCollectionView();
-      this.resultsTabPage = new System.Windows.Forms.TabPage();
-      this.characteristicsViewHost = new HeuristicLab.MainForm.WindowsForms.ViewHost();
+      this.crossvalidationTabPage = new System.Windows.Forms.TabPage();
+      this.xValidateButton = new System.Windows.Forms.Button();
+      this.minTargetView = new HeuristicLab.Data.Views.StringConvertibleValueView();
+      this.minimumTargetLabel = new System.Windows.Forms.Label();
       this.tabControl.SuspendLayout();
       this.characteristicsTabPage.SuspendLayout();
       this.parametersTabPage.SuspendLayout();
+      this.crossvalidationTabPage.SuspendLayout();
       this.SuspendLayout();
       // 
@@ -77 +81 @@
       this.recommenderComboBox.Location = new System.Drawing.Point(90, 3);
       this.recommenderComboBox.Name = "recommenderComboBox";
-      this.recommenderComboBox.Size = new System.Drawing.Size(264, 21);
+      this.recommenderComboBox.Size = new System.Drawing.Size(206, 21);
       this.recommenderComboBox.TabIndex = 11;
       this.recommenderComboBox.SelectedIndexChanged += new System.EventHandler(this.RecommenderComboBoxOnSelectedIndexChanged);
@@ -84 +88 @@
       // 
       this.recommendStartButton.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
-      this.recommendStartButton.Location = new System.Drawing.Point(360, 3);
+      this.recommendStartButton.Location = new System.Drawing.Point(442, 1);
       this.recommendStartButton.Name = "recommendStartButton";
       this.recommendStartButton.Size = new System.Drawing.Size(26, 23);
@@ -99 +103 @@
       this.tabControl.Controls.Add(this.characteristicsTabPage);
       this.tabControl.Controls.Add(this.parametersTabPage);
-      this.tabControl.Controls.Add(this.resultsTabPage);
+      this.tabControl.Controls.Add(this.crossvalidationTabPage);
       this.tabControl.Location = new System.Drawing.Point(0, 30);
       this.tabControl.Name = "tabControl";
@@ -117 +121 @@
       this.characteristicsTabPage.Text = "Characteristics";
       this.characteristicsTabPage.UseVisualStyleBackColor = true;
-      // 
-      // refreshCharacteristicsButton
-      // 
-      this.refreshCharacteristicsButton.Location = new System.Drawing.Point(6, 6);
-      this.refreshCharacteristicsButton.Name = "refreshCharacteristicsButton";
-      this.refreshCharacteristicsButton.Size = new System.Drawing.Size(26, 23);
-      this.refreshCharacteristicsButton.TabIndex = 0;
-      this.refreshCharacteristicsButton.Text = "Refresh";
-      this.refreshCharacteristicsButton.UseVisualStyleBackColor = true;
-      this.refreshCharacteristicsButton.Click += new System.EventHandler(this.RefreshCharacteristicsButtonOnClick);
-      // 
-      // parametersTabPage
-      // 
-      this.parametersTabPage.Controls.Add(this.parameterCollectionView);
-      this.parametersTabPage.Location = new System.Drawing.Point(4, 22);
-      this.parametersTabPage.Name = "parametersTabPage";
-      this.parametersTabPage.Padding = new System.Windows.Forms.Padding(3);
-      this.parametersTabPage.Size = new System.Drawing.Size(692, 324);
-      this.parametersTabPage.TabIndex = 0;
-      this.parametersTabPage.Text = "Parameters";
-      this.parametersTabPage.UseVisualStyleBackColor = true;
-      // 
-      // parameterCollectionView
-      // 
-      this.parameterCollectionView.AllowEditingOfHiddenParameters = true;
-      this.parameterCollectionView.Caption = "ParameterCollection View";
-      this.parameterCollectionView.Content = null;
-      this.parameterCollectionView.Dock = System.Windows.Forms.DockStyle.Fill;
-      this.parameterCollectionView.Location = new System.Drawing.Point(3, 3);
-      this.parameterCollectionView.Name = "parameterCollectionView";
-      this.parameterCollectionView.ReadOnly = false;
-      this.parameterCollectionView.ShowDetails = true;
-      this.parameterCollectionView.Size = new System.Drawing.Size(686, 318);
-      this.parameterCollectionView.TabIndex = 0;
-      // 
-      // resultsTabPage
-      // 
-      this.resultsTabPage.Location = new System.Drawing.Point(4, 22);
-      this.resultsTabPage.Name = "resultsTabPage";
-      this.resultsTabPage.Padding = new System.Windows.Forms.Padding(3);
-      this.resultsTabPage.Size = new System.Drawing.Size(692, 324);
-      this.resultsTabPage.TabIndex = 1;
-      this.resultsTabPage.Text = "Results";
-      this.resultsTabPage.UseVisualStyleBackColor = true;
       // 
       // characteristicsViewHost
@@ -178 +138 @@
       this.characteristicsViewHost.ViewType = null;
       // 
+      // refreshCharacteristicsButton
+      // 
+      this.refreshCharacteristicsButton.Location = new System.Drawing.Point(6, 6);
+      this.refreshCharacteristicsButton.Name = "refreshCharacteristicsButton";
+      this.refreshCharacteristicsButton.Size = new System.Drawing.Size(26, 23);
+      this.refreshCharacteristicsButton.TabIndex = 0;
+      this.refreshCharacteristicsButton.Text = "Refresh";
+      this.refreshCharacteristicsButton.UseVisualStyleBackColor = true;
+      this.refreshCharacteristicsButton.Click += new System.EventHandler(this.RefreshCharacteristicsButtonOnClick);
+      // 
+      // parametersTabPage
+      // 
+      this.parametersTabPage.Controls.Add(this.parameterCollectionView);
+      this.parametersTabPage.Location = new System.Drawing.Point(4, 22);
+      this.parametersTabPage.Name = "parametersTabPage";
+      this.parametersTabPage.Padding = new System.Windows.Forms.Padding(3);
+      this.parametersTabPage.Size = new System.Drawing.Size(692, 324);
+      this.parametersTabPage.TabIndex = 0;
+      this.parametersTabPage.Text = "Parameters";
+      this.parametersTabPage.UseVisualStyleBackColor = true;
+      // 
+      // parameterCollectionView
+      // 
+      this.parameterCollectionView.AllowEditingOfHiddenParameters = true;
+      this.parameterCollectionView.Caption = "ParameterCollection View";
+      this.parameterCollectionView.Content = null;
+      this.parameterCollectionView.Dock = System.Windows.Forms.DockStyle.Fill;
+      this.parameterCollectionView.Location = new System.Drawing.Point(3, 3);
+      this.parameterCollectionView.Name = "parameterCollectionView";
+      this.parameterCollectionView.ReadOnly = false;
+      this.parameterCollectionView.ShowDetails = true;
+      this.parameterCollectionView.Size = new System.Drawing.Size(686, 318);
+      this.parameterCollectionView.TabIndex = 0;
+      // 
+      // crossvalidationTabPage
+      // 
+      this.crossvalidationTabPage.Controls.Add(this.xValidateButton);
+      this.crossvalidationTabPage.Location = new System.Drawing.Point(4, 22);
+      this.crossvalidationTabPage.Name = "crossvalidationTabPage";
+      this.crossvalidationTabPage.Padding = new System.Windows.Forms.Padding(3);
+      this.crossvalidationTabPage.Size = new System.Drawing.Size(692, 324);
+      this.crossvalidationTabPage.TabIndex = 1;
+      this.crossvalidationTabPage.Text = "Crossvalidation";
+      this.crossvalidationTabPage.UseVisualStyleBackColor = true;
+      // 
+      // xValidateButton
+      // 
+      this.xValidateButton.Location = new System.Drawing.Point(7, 7);
+      this.xValidateButton.Name = "xValidateButton";
+      this.xValidateButton.Size = new System.Drawing.Size(163, 23);
+      this.xValidateButton.TabIndex = 0;
+      this.xValidateButton.Text = "X-Validate Leave One Out";
+      this.xValidateButton.UseVisualStyleBackColor = true;
+      this.xValidateButton.Click += new System.EventHandler(this.xValidateButton_Click);
+      // 
+      // minTargetView
+      // 
+      this.minTargetView.Caption = "StringConvertibleValue View";
+      this.minTargetView.Content = null;
+      this.minTargetView.LabelVisible = false;
+      this.minTargetView.Location = new System.Drawing.Point(350, 3);
+      this.minTargetView.Name = "minTargetView";
+      this.minTargetView.ReadOnly = false;
+      this.minTargetView.Size = new System.Drawing.Size(86, 21);
+      this.minTargetView.TabIndex = 18;
+      // 
+      // minimumTargetLabel
+      // 
+      this.minimumTargetLabel.AutoSize = true;
+      this.minimumTargetLabel.Location = new System.Drawing.Point(311, 6);
+      this.minimumTargetLabel.Name = "minimumTargetLabel";
+      this.minimumTargetLabel.Size = new System.Drawing.Size(41, 13);
+      this.minimumTargetLabel.TabIndex = 17;
+      this.minimumTargetLabel.Text = "Target:";
+      // 
       // PerformanceModelingView
       // 
       this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
       this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
+      this.Controls.Add(this.minimumTargetLabel);
+      this.Controls.Add(this.minTargetView);
       this.Controls.Add(this.tabControl);
       this.Controls.Add(this.recommenderLabel);
@@ -190 +227 @@
       this.characteristicsTabPage.ResumeLayout(false);
       this.parametersTabPage.ResumeLayout(false);
+      this.crossvalidationTabPage.ResumeLayout(false);
       this.ResumeLayout(false);
       this.PerformLayout();
@@ -202 +240 @@
     private System.Windows.Forms.TabControl tabControl;
     private System.Windows.Forms.TabPage parametersTabPage;
-    private System.Windows.Forms.TabPage resultsTabPage;
+    private System.Windows.Forms.TabPage crossvalidationTabPage;
     private Core.Views.ParameterCollectionView parameterCollectionView;
     private System.Windows.Forms.TabPage characteristicsTabPage;
     private System.Windows.Forms.Button refreshCharacteristicsButton;
     private MainForm.WindowsForms.ViewHost characteristicsViewHost;
+    private System.Windows.Forms.Button xValidateButton;
+    private Data.Views.StringConvertibleValueView minTargetView;
+    private System.Windows.Forms.Label minimumTargetLabel;
   }
 }

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.cs

@@ -24 +24 @@
 using HeuristicLab.Data;
 using HeuristicLab.MainForm;
+using HeuristicLab.Optimization;
 using HeuristicLab.OptimizationExpertSystem.Common;
 using HeuristicLab.PluginInfrastructure;
@@ -50 +51 @@
     protected override void OnContentChanged() {
       base.OnContentChanged();
+      if (Content == null) {
+        minTargetView.Content = null;
+      } else {
+        minTargetView.Content = Content.MinimumTarget;
+      }
       UpdateCharacteristics();
     }
@@ -58 +64 @@
       recommendStartButton.Enabled = Content != null && recommenderComboBox.SelectedIndex >= 0 && characteristics.CheckedItems.Any();
       characteristicsViewHost.Enabled = Content != null;
+      xValidateButton.Enabled = Content != null && recommenderComboBox.SelectedIndex >= 0 && characteristics.CheckedItems.Any();
     }
 
@@ -82 +89 @@
 
       var @checked = new HashSet<string>(characteristics.CheckedItems.Select(x => x.Value.Value));
+      if (@checked.Count == 0 && Content.ProblemInstances.ResultNames.Any(x => x.StartsWith("Characteristic.")))
+        @checked = new HashSet<string>(Content.ProblemInstances.ResultNames.Where(x => x.StartsWith("Characteristic.")));
+
       characteristics.Clear();
       if (Content == null || Content.ProblemInstances.Count == 0) return;
@@ -109 +119 @@
     private void RecommendStartButtonOnClick(object sender, EventArgs e) {
       var rcmd = (IAlgorithmInstanceRecommender)recommenderComboBox.SelectedItem;
-      Content.RecommendationModel = rcmd.TrainModel(Content, characteristics.CheckedItems.Select(x => x.Value.Value).ToArray());
+      var trainingSet = Content.ProblemInstances.Where(x => !Content.IsCurrentInstance(x)).ToArray();
+      Content.RecommendationModel = rcmd.TrainModel(trainingSet, Content, characteristics.CheckedItems.Select(x => x.Value.Value).ToArray());
     }
 
@@ -115 +126 @@
       UpdateCharacteristics();
       SetEnabledStateOfControls();
+    }
+
+    private void xValidateButton_Click(object sender, EventArgs e) {
+      var recommender = (IAlgorithmInstanceRecommender)recommenderComboBox.SelectedItem;
+      var features = characteristics.CheckedItems.Select(x => x.Value.Value).ToArray();
+
+      var trainingSet = Content.ProblemInstances.Where(x => !Content.IsCurrentInstance(x)).ToArray();
+
+      // leave one out crossvalidation
+      foreach (var pi in trainingSet) {
+        var model = recommender.TrainModel(trainingSet.Where(x => x != pi).ToArray(), Content, features);
+        var ranking = model.GetRanking(pi).ToList();
+        var realPerformance = Content.GetAlgorithmPerformance(pi);
+        // absolute error predicting ert
+        // confusion matrix predicting class
+        // Kendall's tau
+        // average NCDG ... relevance determined by clustering (unsuccessful algorithms being penalized with negative relevance)
+        // mean reciprocal rank
+        // optional: expected reciprocal rank
+      }
+    }
+
+    private static double AbsoluteError(Dictionary<IAlgorithm, double> performance, List<Tuple<IAlgorithm, double>> ranking) {
+      var error = 0.0;
+      foreach (var tuple in ranking) {
+        double actual;
+        if (!performance.TryGetValue(tuple.Item1, out actual)) continue;
+        error += Math.Abs(actual - tuple.Item2);
+      }
+      return error;
+    }
+
+    private static int[,] ConfusionMatrix(Dictionary<IAlgorithm, double> performance, List<Tuple<IAlgorithm, double>> ranking) {
+      var confMatrix = new int[5,5];
+      return confMatrix;
     }
     #endregion

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/SolverView.Designer.cs

@@ -45 +45 @@
     /// </summary>
     private void InitializeComponent() {
-      WeifenLuo.WinFormsUI.Docking.DockPanelSkin dockPanelSkin4 = new WeifenLuo.WinFormsUI.Docking.DockPanelSkin();
-      WeifenLuo.WinFormsUI.Docking.AutoHideStripSkin autoHideStripSkin4 = new WeifenLuo.WinFormsUI.Docking.AutoHideStripSkin();
-      WeifenLuo.WinFormsUI.Docking.DockPanelGradient dockPanelGradient10 = new WeifenLuo.WinFormsUI.Docking.DockPanelGradient();
-      WeifenLuo.WinFormsUI.Docking.TabGradient tabGradient22 = new WeifenLuo.WinFormsUI.Docking.TabGradient();
-      WeifenLuo.WinFormsUI.Docking.DockPaneStripSkin dockPaneStripSkin4 = new WeifenLuo.WinFormsUI.Docking.DockPaneStripSkin();
-      WeifenLuo.WinFormsUI.Docking.DockPaneStripGradient dockPaneStripGradient4 = new WeifenLuo.WinFormsUI.Docking.DockPaneStripGradient();
-      WeifenLuo.WinFormsUI.Docking.TabGradient tabGradient23 = new WeifenLuo.WinFormsUI.Docking.TabGradient();
-      WeifenLuo.WinFormsUI.Docking.DockPanelGradient dockPanelGradient11 = new WeifenLuo.WinFormsUI.Docking.DockPanelGradient();
-      WeifenLuo.WinFormsUI.Docking.TabGradient tabGradient24 = new WeifenLuo.WinFormsUI.Docking.TabGradient();
-      WeifenLuo.WinFormsUI.Docking.DockPaneStripToolWindowGradient dockPaneStripToolWindowGradient4 = new WeifenLuo.WinFormsUI.Docking.DockPaneStripToolWindowGradient();
-      WeifenLuo.WinFormsUI.Docking.TabGradient tabGradient25 = new WeifenLuo.WinFormsUI.Docking.TabGradient();
-      WeifenLuo.WinFormsUI.Docking.TabGradient tabGradient26 = new WeifenLuo.WinFormsUI.Docking.TabGradient();
-      WeifenLuo.WinFormsUI.Docking.DockPanelGradient dockPanelGradient12 = new WeifenLuo.WinFormsUI.Docking.DockPanelGradient();
-      WeifenLuo.WinFormsUI.Docking.TabGradient tabGradient27 = new WeifenLuo.WinFormsUI.Docking.TabGradient();
-      WeifenLuo.WinFormsUI.Docking.TabGradient tabGradient28 = new WeifenLuo.WinFormsUI.Docking.TabGradient();
-      this.minTargetView = new HeuristicLab.Data.Views.StringConvertibleValueView();
+      WeifenLuo.WinFormsUI.Docking.DockPanelSkin dockPanelSkin1 = new WeifenLuo.WinFormsUI.Docking.DockPanelSkin();
+      WeifenLuo.WinFormsUI.Docking.AutoHideStripSkin autoHideStripSkin1 = new WeifenLuo.WinFormsUI.Docking.AutoHideStripSkin();
+      WeifenLuo.WinFormsUI.Docking.DockPanelGradient dockPanelGradient1 = new WeifenLuo.WinFormsUI.Docking.DockPanelGradient();
+      WeifenLuo.WinFormsUI.Docking.TabGradient tabGradient1 = new WeifenLuo.WinFormsUI.Docking.TabGradient();
+      WeifenLuo.WinFormsUI.Docking.DockPaneStripSkin dockPaneStripSkin1 = new WeifenLuo.WinFormsUI.Docking.DockPaneStripSkin();
+      WeifenLuo.WinFormsUI.Docking.DockPaneStripGradient dockPaneStripGradient1 = new WeifenLuo.WinFormsUI.Docking.DockPaneStripGradient();
+      WeifenLuo.WinFormsUI.Docking.TabGradient tabGradient2 = new WeifenLuo.WinFormsUI.Docking.TabGradient();
+      WeifenLuo.WinFormsUI.Docking.DockPanelGradient dockPanelGradient2 = new WeifenLuo.WinFormsUI.Docking.DockPanelGradient();
+      WeifenLuo.WinFormsUI.Docking.TabGradient tabGradient3 = new WeifenLuo.WinFormsUI.Docking.TabGradient();
+      WeifenLuo.WinFormsUI.Docking.DockPaneStripToolWindowGradient dockPaneStripToolWindowGradient1 = new WeifenLuo.WinFormsUI.Docking.DockPaneStripToolWindowGradient();
+      WeifenLuo.WinFormsUI.Docking.TabGradient tabGradient4 = new WeifenLuo.WinFormsUI.Docking.TabGradient();
+      WeifenLuo.WinFormsUI.Docking.TabGradient tabGradient5 = new WeifenLuo.WinFormsUI.Docking.TabGradient();
+      WeifenLuo.WinFormsUI.Docking.DockPanelGradient dockPanelGradient3 = new WeifenLuo.WinFormsUI.Docking.DockPanelGradient();
+      WeifenLuo.WinFormsUI.Docking.TabGradient tabGradient6 = new WeifenLuo.WinFormsUI.Docking.TabGradient();
+      WeifenLuo.WinFormsUI.Docking.TabGradient tabGradient7 = new WeifenLuo.WinFormsUI.Docking.TabGradient();
       this.maxEvaluationsView = new HeuristicLab.Data.Views.StringConvertibleValueView();
       this.algorithmCloneButton = new System.Windows.Forms.Button();
       this.seedingStrategyPanel = new System.Windows.Forms.Panel();
       this.algorithmStartButton = new System.Windows.Forms.Button();
-      this.minimumTargetLabel = new System.Windows.Forms.Label();
       this.seedingStrategyLabel = new System.Windows.Forms.Label();
       this.evaluationsLimitabel = new System.Windows.Forms.Label();
@@ -91 +89 @@
       this.SuspendLayout();
       // 
-      // minTargetView
-      // 
-      this.minTargetView.Caption = "StringConvertibleValue View";
-      this.minTargetView.Content = null;
-      this.minTargetView.LabelVisible = false;
-      this.minTargetView.Location = new System.Drawing.Point(431, 3);
-      this.minTargetView.Name = "minTargetView";
-      this.minTargetView.ReadOnly = false;
-      this.minTargetView.Size = new System.Drawing.Size(255, 21);
-      this.minTargetView.TabIndex = 16;
-      // 
       // maxEvaluationsView
       // 
@@ -142 +129 @@
       this.algorithmStartButton.Click += new System.EventHandler(this.AlgorithmStartButtonOnClick);
       // 
-      // minimumTargetLabel
-      // 
-      this.minimumTargetLabel.AutoSize = true;
-      this.minimumTargetLabel.Location = new System.Drawing.Point(384, 6);
-      this.minimumTargetLabel.Name = "minimumTargetLabel";
-      this.minimumTargetLabel.Size = new System.Drawing.Size(41, 13);
-      this.minimumTargetLabel.TabIndex = 13;
-      this.minimumTargetLabel.Text = "Target:";
-      // 
       // seedingStrategyLabel
       // 
@@ -190 +168 @@
       // solverTabControl
       // 
+      this.solverTabControl.AllowDrop = true;
       this.solverTabControl.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) 
             | System.Windows.Forms.AnchorStyles.Left) 
@@ -238 +217 @@
       this.resultsDockPanel.Name = "resultsDockPanel";
       this.resultsDockPanel.Size = new System.Drawing.Size(829, 421);
-      dockPanelGradient10.EndColor = System.Drawing.SystemColors.ControlLight;
-      dockPanelGradient10.StartColor = System.Drawing.SystemColors.ControlLight;
-      autoHideStripSkin4.DockStripGradient = dockPanelGradient10;
-      tabGradient22.EndColor = System.Drawing.SystemColors.Control;
-      tabGradient22.StartColor = System.Drawing.SystemColors.Control;
-      tabGradient22.TextColor = System.Drawing.SystemColors.ControlDarkDark;
-      autoHideStripSkin4.TabGradient = tabGradient22;
-      autoHideStripSkin4.TextFont = new System.Drawing.Font("Segoe UI", 9F);
-      dockPanelSkin4.AutoHideStripSkin = autoHideStripSkin4;
-      tabGradient23.EndColor = System.Drawing.SystemColors.ControlLightLight;
-      tabGradient23.StartColor = System.Drawing.SystemColors.ControlLightLight;
-      tabGradient23.TextColor = System.Drawing.SystemColors.ControlText;
-      dockPaneStripGradient4.ActiveTabGradient = tabGradient23;
-      dockPanelGradient11.EndColor = System.Drawing.SystemColors.Control;
-      dockPanelGradient11.StartColor = System.Drawing.SystemColors.Control;
-      dockPaneStripGradient4.DockStripGradient = dockPanelGradient11;
-      tabGradient24.EndColor = System.Drawing.SystemColors.ControlLight;
-      tabGradient24.StartColor = System.Drawing.SystemColors.ControlLight;
-      tabGradient24.TextColor = System.Drawing.SystemColors.ControlText;
-      dockPaneStripGradient4.InactiveTabGradient = tabGradient24;
-      dockPaneStripSkin4.DocumentGradient = dockPaneStripGradient4;
-      dockPaneStripSkin4.TextFont = new System.Drawing.Font("Segoe UI", 9F);
-      tabGradient25.EndColor = System.Drawing.SystemColors.ActiveCaption;
-      tabGradient25.LinearGradientMode = System.Drawing.Drawing2D.LinearGradientMode.Vertical;
-      tabGradient25.StartColor = System.Drawing.SystemColors.GradientActiveCaption;
-      tabGradient25.TextColor = System.Drawing.SystemColors.ActiveCaptionText;
-      dockPaneStripToolWindowGradient4.ActiveCaptionGradient = tabGradient25;
-      tabGradient26.EndColor = System.Drawing.SystemColors.Control;
-      tabGradient26.StartColor = System.Drawing.SystemColors.Control;
-      tabGradient26.TextColor = System.Drawing.SystemColors.ControlText;
-      dockPaneStripToolWindowGradient4.ActiveTabGradient = tabGradient26;
-      dockPanelGradient12.EndColor = System.Drawing.SystemColors.ControlLight;
-      dockPanelGradient12.StartColor = System.Drawing.SystemColors.ControlLight;
-      dockPaneStripToolWindowGradient4.DockStripGradient = dockPanelGradient12;
-      tabGradient27.EndColor = System.Drawing.SystemColors.InactiveCaption;
-      tabGradient27.LinearGradientMode = System.Drawing.Drawing2D.LinearGradientMode.Vertical;
-      tabGradient27.StartColor = System.Drawing.SystemColors.GradientInactiveCaption;
-      tabGradient27.TextColor = System.Drawing.SystemColors.InactiveCaptionText;
-      dockPaneStripToolWindowGradient4.InactiveCaptionGradient = tabGradient27;
-      tabGradient28.EndColor = System.Drawing.Color.Transparent;
-      tabGradient28.StartColor = System.Drawing.Color.Transparent;
-      tabGradient28.TextColor = System.Drawing.SystemColors.ControlDarkDark;
-      dockPaneStripToolWindowGradient4.InactiveTabGradient = tabGradient28;
-      dockPaneStripSkin4.ToolWindowGradient = dockPaneStripToolWindowGradient4;
-      dockPanelSkin4.DockPaneStripSkin = dockPaneStripSkin4;
-      this.resultsDockPanel.Skin = dockPanelSkin4;
+      dockPanelGradient1.EndColor = System.Drawing.SystemColors.ControlLight;
+      dockPanelGradient1.StartColor = System.Drawing.SystemColors.ControlLight;
+      autoHideStripSkin1.DockStripGradient = dockPanelGradient1;
+      tabGradient1.EndColor = System.Drawing.SystemColors.Control;
+      tabGradient1.StartColor = System.Drawing.SystemColors.Control;
+      tabGradient1.TextColor = System.Drawing.SystemColors.ControlDarkDark;
+      autoHideStripSkin1.TabGradient = tabGradient1;
+      autoHideStripSkin1.TextFont = new System.Drawing.Font("Segoe UI", 9F);
+      dockPanelSkin1.AutoHideStripSkin = autoHideStripSkin1;
+      tabGradient2.EndColor = System.Drawing.SystemColors.ControlLightLight;
+      tabGradient2.StartColor = System.Drawing.SystemColors.ControlLightLight;
+      tabGradient2.TextColor = System.Drawing.SystemColors.ControlText;
+      dockPaneStripGradient1.ActiveTabGradient = tabGradient2;
+      dockPanelGradient2.EndColor = System.Drawing.SystemColors.Control;
+      dockPanelGradient2.StartColor = System.Drawing.SystemColors.Control;
+      dockPaneStripGradient1.DockStripGradient = dockPanelGradient2;
+      tabGradient3.EndColor = System.Drawing.SystemColors.ControlLight;
+      tabGradient3.StartColor = System.Drawing.SystemColors.ControlLight;
+      tabGradient3.TextColor = System.Drawing.SystemColors.ControlText;
+      dockPaneStripGradient1.InactiveTabGradient = tabGradient3;
+      dockPaneStripSkin1.DocumentGradient = dockPaneStripGradient1;
+      dockPaneStripSkin1.TextFont = new System.Drawing.Font("Segoe UI", 9F);
+      tabGradient4.EndColor = System.Drawing.SystemColors.ActiveCaption;
+      tabGradient4.LinearGradientMode = System.Drawing.Drawing2D.LinearGradientMode.Vertical;
+      tabGradient4.StartColor = System.Drawing.SystemColors.GradientActiveCaption;
+      tabGradient4.TextColor = System.Drawing.SystemColors.ActiveCaptionText;
+      dockPaneStripToolWindowGradient1.ActiveCaptionGradient = tabGradient4;
+      tabGradient5.EndColor = System.Drawing.SystemColors.Control;
+      tabGradient5.StartColor = System.Drawing.SystemColors.Control;
+      tabGradient5.TextColor = System.Drawing.SystemColors.ControlText;
+      dockPaneStripToolWindowGradient1.ActiveTabGradient = tabGradient5;
+      dockPanelGradient3.EndColor = System.Drawing.SystemColors.ControlLight;
+      dockPanelGradient3.StartColor = System.Drawing.SystemColors.ControlLight;
+      dockPaneStripToolWindowGradient1.DockStripGradient = dockPanelGradient3;
+      tabGradient6.EndColor = System.Drawing.SystemColors.InactiveCaption;
+      tabGradient6.LinearGradientMode = System.Drawing.Drawing2D.LinearGradientMode.Vertical;
+      tabGradient6.StartColor = System.Drawing.SystemColors.GradientInactiveCaption;
+      tabGradient6.TextColor = System.Drawing.SystemColors.InactiveCaptionText;
+      dockPaneStripToolWindowGradient1.InactiveCaptionGradient = tabGradient6;
+      tabGradient7.EndColor = System.Drawing.Color.Transparent;
+      tabGradient7.StartColor = System.Drawing.Color.Transparent;
+      tabGradient7.TextColor = System.Drawing.SystemColors.ControlDarkDark;
+      dockPaneStripToolWindowGradient1.InactiveTabGradient = tabGradient7;
+      dockPaneStripSkin1.ToolWindowGradient = dockPaneStripToolWindowGradient1;
+      dockPanelSkin1.DockPaneStripSkin = dockPaneStripSkin1;
+      this.resultsDockPanel.Skin = dockPanelSkin1;
       this.resultsDockPanel.TabIndex = 0;
       // 
@@ -292 +271 @@
       this.runsTabPage.Name = "runsTabPage";
       this.runsTabPage.Padding = new System.Windows.Forms.Padding(3);
-      this.runsTabPage.Size = new System.Drawing.Size(548, 418);
+      this.runsTabPage.Size = new System.Drawing.Size(835, 453);
       this.runsTabPage.TabIndex = 4;
       this.runsTabPage.Text = "Runs";
@@ -305 +284 @@
       this.runsView.Name = "runsView";
       this.runsView.ReadOnly = false;
-      this.runsView.Size = new System.Drawing.Size(542, 412);
+      this.runsView.Size = new System.Drawing.Size(829, 447);
       this.runsView.TabIndex = 0;
       // 
@@ -314 +293 @@
       this.seededRunsTabPage.Name = "seededRunsTabPage";
       this.seededRunsTabPage.Padding = new System.Windows.Forms.Padding(3);
-      this.seededRunsTabPage.Size = new System.Drawing.Size(548, 418);
+      this.seededRunsTabPage.Size = new System.Drawing.Size(835, 453);
       this.seededRunsTabPage.TabIndex = 5;
       this.seededRunsTabPage.Text = "Seeded Runs";
@@ -327 +306 @@
       this.seededRunsView.Name = "seededRunsView";
       this.seededRunsView.ReadOnly = false;
-      this.seededRunsView.Size = new System.Drawing.Size(542, 412);
+      this.seededRunsView.Size = new System.Drawing.Size(829, 447);
       this.seededRunsView.TabIndex = 1;
       // 
@@ -335 +314 @@
       this.solutionSeedingTabPage.Name = "solutionSeedingTabPage";
       this.solutionSeedingTabPage.Padding = new System.Windows.Forms.Padding(3);
-      this.solutionSeedingTabPage.Size = new System.Drawing.Size(548, 418);
+      this.solutionSeedingTabPage.Size = new System.Drawing.Size(835, 453);
       this.solutionSeedingTabPage.TabIndex = 1;
       this.solutionSeedingTabPage.Text = "Seeding Pool";
@@ -346 +325 @@
       this.parametersTabPage.Name = "parametersTabPage";
       this.parametersTabPage.Padding = new System.Windows.Forms.Padding(3);
-      this.parametersTabPage.Size = new System.Drawing.Size(548, 418);
+      this.parametersTabPage.Size = new System.Drawing.Size(835, 453);
       this.parametersTabPage.TabIndex = 0;
       this.parametersTabPage.Text = "Parameters";
@@ -361 +340 @@
       this.solverParametersView.ReadOnly = true;
       this.solverParametersView.ShowDetails = true;
-      this.solverParametersView.Size = new System.Drawing.Size(542, 412);
+      this.solverParametersView.Size = new System.Drawing.Size(829, 447);
       this.solverParametersView.TabIndex = 0;
       // 
@@ -370 +349 @@
       this.operatorGraphTabPage.Name = "operatorGraphTabPage";
       this.operatorGraphTabPage.Padding = new System.Windows.Forms.Padding(3);
-      this.operatorGraphTabPage.Size = new System.Drawing.Size(548, 418);
+      this.operatorGraphTabPage.Size = new System.Drawing.Size(835, 453);
       this.operatorGraphTabPage.TabIndex = 3;
       this.operatorGraphTabPage.Text = "Operator Graph";
@@ -384 +363 @@
       this.operatorGraphViewHost.Name = "operatorGraphViewHost";
       this.operatorGraphViewHost.ReadOnly = true;
-      this.operatorGraphViewHost.Size = new System.Drawing.Size(542, 412);
+      this.operatorGraphViewHost.Size = new System.Drawing.Size(829, 447);
       this.operatorGraphViewHost.TabIndex = 0;
       this.operatorGraphViewHost.ViewsLabelVisible = true;
@@ -397 +376 @@
       this.Controls.Add(this.instanceLabel);
       this.Controls.Add(this.algorithmStartButton);
-      this.Controls.Add(this.minTargetView);
       this.Controls.Add(this.maxEvaluationsView);
       this.Controls.Add(this.suggestedInstancesComboBox);
       this.Controls.Add(this.seedingStrategyPanel);
-      this.Controls.Add(this.minimumTargetLabel);
       this.Controls.Add(this.seedingStrategyLabel);
       this.Controls.Add(this.evaluationsLimitabel);
@@ -427 +404 @@
     private System.Windows.Forms.Button algorithmCloneButton;
     private Data.Views.StringConvertibleValueView maxEvaluationsView;
-    private System.Windows.Forms.Label minimumTargetLabel;
-    private Data.Views.StringConvertibleValueView minTargetView;
     private System.Windows.Forms.Label instanceLabel;
     private MainForm.WindowsForms.DragOverTabControl solverTabControl;

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/SolverView.cs

@@ -61 +61 @@
         if (Content == null) {
           maxEvaluationsView.Content = null;
-          minTargetView.Content = null;
           solverParametersView.Content = null;
           runsView.Content = null;
@@ -69 +68 @@
         } else {
           maxEvaluationsView.Content = Content.MaximumEvaluations;
-          minTargetView.Content = Content.MinimumTarget;
           runsView.Content = Content.InstanceRuns;
           seededRunsView.Content = Content.SeededRuns;
@@ -155 +153 @@
 
       public override string ToString() {
-        return item.Item2.ToString("F0") + ": " + item.Item1.Name;
+        return item.Item2.ToString("N0") + ": " + item.Item1.Name;
       }
     }

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/UnderstandingProblemInstanceView.Designer.cs

@@ -54 +54 @@
       this.mapSplitContainer = new System.Windows.Forms.SplitContainer();
       this.showCharacteristicsCheckBox = new System.Windows.Forms.CheckBox();
-      this.fromZeroCheckBox = new System.Windows.Forms.CheckBox();
       this.invPropCheckBox = new System.Windows.Forms.CheckBox();
+      this.colorLabel = new System.Windows.Forms.Label();
       this.sizeLabel = new System.Windows.Forms.Label();
       this.projectionLabel = new System.Windows.Forms.Label();
       this.instanceMapChart = new HeuristicLab.Visualization.ChartControlsExtensions.EnhancedChart();
+      this.colorComboBox = new System.Windows.Forms.ComboBox();
       this.sizeComboBox = new System.Windows.Forms.ComboBox();
       this.projectionComboBox = new System.Windows.Forms.ComboBox();
+      this.characteristicsPanel = new System.Windows.Forms.Panel();
+      this.updateProjectionButton = new System.Windows.Forms.Button();
       this.instancesTabPage = new System.Windows.Forms.TabPage();
       this.problemInstancesView = new HeuristicLab.MainForm.WindowsForms.ViewHost();
-      this.colorComboBox = new System.Windows.Forms.ComboBox();
-      this.colorLabel = new System.Windows.Forms.Label();
       this.problemInstancesTabControl.SuspendLayout();
       this.mapTabPage.SuspendLayout();
       ((System.ComponentModel.ISupportInitialize)(this.mapSplitContainer)).BeginInit();
       this.mapSplitContainer.Panel1.SuspendLayout();
+      this.mapSplitContainer.Panel2.SuspendLayout();
       this.mapSplitContainer.SuspendLayout();
       ((System.ComponentModel.ISupportInitialize)(this.instanceMapChart)).BeginInit();
@@ -109 +111 @@
       // 
       this.mapSplitContainer.Panel1.Controls.Add(this.showCharacteristicsCheckBox);
-      this.mapSplitContainer.Panel1.Controls.Add(this.fromZeroCheckBox);
       this.mapSplitContainer.Panel1.Controls.Add(this.invPropCheckBox);
       this.mapSplitContainer.Panel1.Controls.Add(this.colorLabel);
@@ -119 +120 @@
       this.mapSplitContainer.Panel1.Controls.Add(this.projectionComboBox);
       this.mapSplitContainer.Panel1.Padding = new System.Windows.Forms.Padding(0, 3, 0, 0);
+      // 
+      // mapSplitContainer.Panel2
+      // 
+      this.mapSplitContainer.Panel2.Controls.Add(this.characteristicsPanel);
+      this.mapSplitContainer.Panel2.Controls.Add(this.updateProjectionButton);
       this.mapSplitContainer.Panel2Collapsed = true;
       this.mapSplitContainer.Size = new System.Drawing.Size(1094, 728);
-      this.mapSplitContainer.SplitterDistance = 847;
+      this.mapSplitContainer.SplitterDistance = 843;
       this.mapSplitContainer.TabIndex = 12;
       // 
@@ -135 +141 @@
       this.showCharacteristicsCheckBox.UseVisualStyleBackColor = true;
       this.showCharacteristicsCheckBox.CheckedChanged += new System.EventHandler(this.showCharacteristicsCheckBox_CheckedChanged);
-      // 
-      // fromZeroCheckBox
-      // 
-      this.fromZeroCheckBox.AutoSize = true;
-      this.fromZeroCheckBox.Checked = true;
-      this.fromZeroCheckBox.CheckState = System.Windows.Forms.CheckState.Checked;
-      this.fromZeroCheckBox.Location = new System.Drawing.Point(639, 8);
-      this.fromZeroCheckBox.Name = "fromZeroCheckBox";
-      this.fromZeroCheckBox.Size = new System.Drawing.Size(69, 17);
-      this.fromZeroCheckBox.TabIndex = 17;
-      this.fromZeroCheckBox.Text = "from zero";
-      this.fromZeroCheckBox.UseVisualStyleBackColor = true;
-      this.fromZeroCheckBox.CheckedChanged += new System.EventHandler(this.fromZeroCheckBox_CheckedChanged);
       // 
       // invPropCheckBox
@@ -161 +154 @@
       this.invPropCheckBox.UseVisualStyleBackColor = true;
       this.invPropCheckBox.CheckedChanged += new System.EventHandler(this.InvPropCheckBoxOnCheckedChanged);
+      // 
+      // colorLabel
+      // 
+      this.colorLabel.AutoSize = true;
+      this.colorLabel.Location = new System.Drawing.Point(652, 9);
+      this.colorLabel.Name = "colorLabel";
+      this.colorLabel.Size = new System.Drawing.Size(34, 13);
+      this.colorLabel.TabIndex = 15;
+      this.colorLabel.Text = "Color:";
       // 
       // sizeLabel
@@ -202 +204 @@
       series1.MarkerSize = 10;
       series1.Name = "InstancesSeries";
+      series1.SmartLabelStyle.CalloutLineAnchorCapStyle = System.Windows.Forms.DataVisualization.Charting.LineAnchorCapStyle.None;
       series2.ChartArea = "Default";
       series2.ChartType = System.Windows.Forms.DataVisualization.Charting.SeriesChartType.Point;
@@ -209 +212 @@
       series2.MarkerStyle = System.Windows.Forms.DataVisualization.Charting.MarkerStyle.Cross;
       series2.Name = "CurrentInstanceSeries";
+      series2.SmartLabelStyle.CalloutLineAnchorCapStyle = System.Windows.Forms.DataVisualization.Charting.LineAnchorCapStyle.None;
       this.instanceMapChart.Series.Add(series1);
       this.instanceMapChart.Series.Add(series2);
       this.instanceMapChart.Size = new System.Drawing.Size(974, 692);
       this.instanceMapChart.TabIndex = 12;
+      // 
+      // colorComboBox
+      // 
+      this.colorComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
+      this.colorComboBox.FormattingEnabled = true;
+      this.colorComboBox.Location = new System.Drawing.Point(692, 6);
+      this.colorComboBox.Name = "colorComboBox";
+      this.colorComboBox.Size = new System.Drawing.Size(222, 21);
+      this.colorComboBox.TabIndex = 13;
+      this.colorComboBox.SelectedIndexChanged += new System.EventHandler(this.colorComboBox_SelectedIndexChanged);
       // 
       // sizeComboBox
@@ -234 +248 @@
       this.projectionComboBox.SelectedIndexChanged += new System.EventHandler(this.ProjectionComboBoxOnSelectedIndexChanged);
       // 
+      // characteristicsPanel
+      // 
+      this.characteristicsPanel.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) 
+            | System.Windows.Forms.AnchorStyles.Left) 
+            | System.Windows.Forms.AnchorStyles.Right)));
+      this.characteristicsPanel.Location = new System.Drawing.Point(3, 33);
+      this.characteristicsPanel.Name = "characteristicsPanel";
+      this.characteristicsPanel.Size = new System.Drawing.Size(241, 692);
+      this.characteristicsPanel.TabIndex = 20;
+      // 
+      // updateProjectionButton
+      // 
+      this.updateProjectionButton.Location = new System.Drawing.Point(3, 6);
+      this.updateProjectionButton.Name = "updateProjectionButton";
+      this.updateProjectionButton.Size = new System.Drawing.Size(26, 23);
+      this.updateProjectionButton.TabIndex = 19;
+      this.updateProjectionButton.Text = "Update";
+      this.updateProjectionButton.UseVisualStyleBackColor = true;
+      this.updateProjectionButton.Click += new System.EventHandler(this.updateProjectionButton_Click);
+      // 
       // instancesTabPage
       // 
@@ -240 +274 @@
       this.instancesTabPage.Name = "instancesTabPage";
       this.instancesTabPage.Padding = new System.Windows.Forms.Padding(3);
-      this.instancesTabPage.Size = new System.Drawing.Size(1106, 734);
+      this.instancesTabPage.Size = new System.Drawing.Size(1100, 734);
       this.instancesTabPage.TabIndex = 0;
       this.instancesTabPage.Text = "Instances";
@@ -254 +288 @@
       this.problemInstancesView.Name = "problemInstancesView";
       this.problemInstancesView.ReadOnly = false;
-      this.problemInstancesView.Size = new System.Drawing.Size(1100, 728);
+      this.problemInstancesView.Size = new System.Drawing.Size(1094, 728);
       this.problemInstancesView.TabIndex = 0;
       this.problemInstancesView.ViewsLabelVisible = true;
       this.problemInstancesView.ViewType = null;
-      // 
-      // colorComboBox
-      // 
-      this.colorComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
-      this.colorComboBox.FormattingEnabled = true;
-      this.colorComboBox.Location = new System.Drawing.Point(768, 6);
-      this.colorComboBox.Name = "colorComboBox";
-      this.colorComboBox.Size = new System.Drawing.Size(222, 21);
-      this.colorComboBox.TabIndex = 13;
-      this.colorComboBox.SelectedIndexChanged += new System.EventHandler(this.colorComboBox_SelectedIndexChanged);
-      // 
-      // colorLabel
-      // 
-      this.colorLabel.AutoSize = true;
-      this.colorLabel.Location = new System.Drawing.Point(728, 9);
-      this.colorLabel.Name = "colorLabel";
-      this.colorLabel.Size = new System.Drawing.Size(34, 13);
-      this.colorLabel.TabIndex = 15;
-      this.colorLabel.Text = "Color:";
       // 
       // UnderstandingProblemInstanceView
@@ -289 +304 @@
       this.mapSplitContainer.Panel1.ResumeLayout(false);
       this.mapSplitContainer.Panel1.PerformLayout();
+      this.mapSplitContainer.Panel2.ResumeLayout(false);
       ((System.ComponentModel.ISupportInitialize)(this.mapSplitContainer)).EndInit();
       this.mapSplitContainer.ResumeLayout(false);
@@ -311 +327 @@
     private System.Windows.Forms.ComboBox sizeComboBox;
     private System.Windows.Forms.ComboBox projectionComboBox;
-    private System.Windows.Forms.CheckBox fromZeroCheckBox;
     private System.Windows.Forms.Label colorLabel;
     private System.Windows.Forms.ComboBox colorComboBox;
+    private System.Windows.Forms.Button updateProjectionButton;
+    private System.Windows.Forms.Panel characteristicsPanel;
   }
 }

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/UnderstandingProblemInstanceView.cs

@@ -45 +45 @@
       showCharacteristicsCheckBox.Text = string.Empty;
       showCharacteristicsCheckBox.Image = VSImageLibrary.Properties;
+      updateProjectionButton.Text = string.Empty;
+      updateProjectionButton.Image = VSImageLibrary.Refresh;
       characteristics = new CheckedItemList<StringValue>();
       characteristics.CheckedItemsChanged += CharacteristicsOnCheckedItemsChanged;
-      mapSplitContainer.Panel2.Controls.Add(new ViewHost() {
+      characteristicsPanel.Controls.Add(new ViewHost() {
         Dock = DockStyle.Fill,
         Content = characteristics
@@ -55 +57 @@
     protected override void OnContentChanged() {
       base.OnContentChanged();
-      characteristics.Clear();
       if (Content == null) {
         problemInstancesView.Content = null;
-        instanceMapChart.Series["InstancesSeries"].Points.Clear();
-        instanceMapChart.Series["CurrentInstanceSeries"].Points.Clear();
       } else {
         problemInstancesView.Content = Content.ProblemInstances;
-        UpdateCharacteristics();
-        UpdateProjectionComboBox();
-        UpdateSizeComboBox();
-        UpdateColorComboBox();
-      }
+      }
+      UpdateCharacteristics();
+      UpdateProjectionComboBox();
+      UpdateSizeComboBox();
+      UpdateColorComboBox();
     }
 
     protected override void SetEnabledStateOfControls() {
       base.SetEnabledStateOfControls();
+      updateProjectionButton.Enabled = Content != null && characteristics.CheckedItems.Any();
     }
 
     #region Update Controls
     private void UpdateCharacteristics() {
-      var @checked = new HashSet<string>(characteristics.CheckedItems.Select(x => x.Value.Value));
-      characteristics.Clear();
-      foreach (var c in Content.ProblemInstances.ResultNames) {
-        characteristics.Add(new StringValue(c), @checked.Contains(c));
-      }
+      if (InvokeRequired) { Invoke((Action)UpdateCharacteristics); return; }
+      try {
+        SuppressEvents = true;
+        var @checked = new HashSet<string>(characteristics.CheckedItems.Select(x => x.Value.Value));
+        characteristics.Clear();
+
+        if (Content == null) return;
+        if (@checked.Count == 0 && Content.ProblemInstances.ResultNames.Any(x => x.StartsWith("Characteristic.")))
+          @checked = new HashSet<string>(Content.ProblemInstances.ResultNames.Where(x => x.StartsWith("Characteristic.")));
+
+        foreach (var c in Content.ProblemInstances.ResultNames) {
+          characteristics.Add(new StringValue(c), @checked.Contains(c));
+        }
+      } finally { SuppressEvents = false; }
     }
 
     private void UpdateProjectionComboBox() {
+      if (InvokeRequired) { Invoke((Action)UpdateProjectionComboBox); return; }
       try {
         SuppressEvents = true;
@@ -97 +107 @@
 
     private void UpdateSizeComboBox() {
+      if (InvokeRequired) { Invoke((Action)UpdateSizeComboBox); return; }
       try {
         SuppressEvents = true;
         var selected = sizeComboBox.SelectedIndex >= 0 ? (string)sizeComboBox.SelectedItem : null;
         sizeComboBox.Items.Clear();
+        
+        if (Content == null) return;
+
         sizeComboBox.Items.Add(string.Empty);
         foreach (var str in Content.ProblemInstances.ResultNames.Where(x => !(x.StartsWith("Projection.") && (x.EndsWith(".X") || x.EndsWith(".Y"))))) {
@@ -112 +126 @@
 
     private void UpdateColorComboBox() {
+      if (InvokeRequired) { Invoke((Action)UpdateColorComboBox); return; }
       try {
         SuppressEvents = true;
         var selected = colorComboBox.SelectedIndex >= 0 ? (string)colorComboBox.SelectedItem : null;
         colorComboBox.Items.Clear();
+
+        if (Content == null) return;
+
         colorComboBox.Items.Add(string.Empty);
         foreach (var str in Content.ProblemInstances.ResultNames.Where(x => x.StartsWith("Rank."))) {
@@ -132 +150 @@
       var currentInstanceSeries = instanceMapChart.Series["CurrentInstanceSeries"];
 
-      if (projectionComboBox.SelectedIndex < 0) {
+      if (projectionComboBox.SelectedIndex < 0 || Content == null) {
         instancesSeries.Points.Clear();
         currentInstanceSeries.Points.Clear();
@@ -142 +160 @@
       var color = colorComboBox.SelectedIndex >= 0 ? (string)colorComboBox.SelectedItem : string.Empty;
 
-      DoProjectProblemInstances(instancesSeries, currentInstanceSeries, projection,
-        size, color, invPropCheckBox.Checked, fromZeroCheckBox.Checked);
+      DoProjectProblemInstances(instancesSeries, currentInstanceSeries, projection, size, color, invPropCheckBox.Checked);
     }
     #endregion
@@ -156 +173 @@
       base.OnProblemInstancesChanged();
       if (Content.ProblemInstances.UpdateOfRunsInProgress) return;
-      try {
-        SuppressEvents = true;
-        UpdateCharacteristics();
-      } finally { SuppressEvents = false; }
+      UpdateCharacteristics();
       UpdateProjectionComboBox();
       UpdateSizeComboBox();
@@ -184 +198 @@
     }
 
-    private void fromZeroCheckBox_CheckedChanged(object sender, EventArgs e) {
-      UpdateProjection();
-    }
-
     private void showCharacteristicsCheckBox_CheckedChanged(object sender, EventArgs e) {
       mapSplitContainer.Panel2Collapsed = !showCharacteristicsCheckBox.Checked;
     }
+
+    private void updateProjectionButton_Click(object sender, EventArgs e) {
+      Content.UpdateInstanceProjection(characteristics.CheckedItems.Select(x => x.Value.Value).ToArray());
+    }
     #endregion
 
     #region Other Event Handlers
     private void CharacteristicsOnCheckedItemsChanged(object sender, EventArgs e) {
-      if (SuppressEvents) return;
-      if (characteristics.CheckedItems.Any())
-        Content.UpdateInstanceProjection(characteristics.CheckedItems.Select(x => x.Value.Value).ToArray());
+      SetEnabledStateOfControls();
     }
     #endregion
@@ -203 +215 @@
     #region Helper Classes and Methods
     private IEnumerable<string> GetProjections() {
+      if (Content == null) return new string[0];
       return Content.ProblemInstances.ResultNames
         .Where(x => Regex.IsMatch(x, "^Projection[.].*[.][XY]$"))
@@ -209 +222 @@
     }
 
-    private void DoProjectProblemInstances(Series instancesSeries, Series currentInstanceSeries, string projection, string size, string color, bool invProp, bool fromZero) {
+    private void DoProjectProblemInstances(Series instancesSeries, Series currentInstanceSeries, string projection, string size, string color, bool invProp) {
       instancesSeries.Points.Clear();
       currentInstanceSeries.Points.Clear();
@@ -215 +228 @@
       double maxSize = 0, minSize = 0;
       if (!string.IsNullOrEmpty(size)) {
-        var sizes = Content.ProblemInstances.Where(x => x.Results.ContainsKey(size)).Select(x => x.Results[size]).OfType<DoubleValue>().Where(x => !double.IsNaN(x.Value)).ToList();
+        var sizes = Content.ProblemInstances
+                           .Where(x => x.Results.ContainsKey(size))
+                           .Select(x => x.Results[size])
+                           .Select(x => x is DoubleValue ? ((DoubleValue)x).Value : (x is IntValue ? ((IntValue)x).Value : double.NaN))
+                           .Where(x => !double.IsNaN(x)).ToList();
         if (sizes.Count > 0) {
-          maxSize = sizes.Max(x => x.Value);
-          if (maxSize < 0 && fromZero) {
+          maxSize = sizes.Max();
+          if (maxSize < 0) {
             maxSize = 0;
-            minSize = sizes.Min(x => x.Value);
-          } else if (fromZero) {
-            minSize = 0;
+            minSize = sizes.Min();
           } else {
-            minSize = sizes.Min(x => x.Value);
+            minSize = sizes.Min();
           }
         }