Changeset 15064 for branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO

Timestamp:

06/26/17 09:10:56 (7 years ago)

Author:

bwerth

Message:

#2745 implemented EGO as EngineAlgorithm + some simplifications in the IInfillCriterion interface

Location:

branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO

Files:

• AdaptiveSamplingAlgorithm.cs (added)
• EfficientGlobalOptimizationAlgorithm.cs (modified) (21 diffs)
• EgoUtilities.cs (modified) (7 diffs)
• HeuristicLab.Algorithms.EGO-3.4.csproj (modified) (9 diffs)
• InfillCriteria/AugmentedExpectedImprovement.cs (modified) (2 diffs)
• InfillCriteria/ExpectedImprovement.cs (modified) (3 diffs)
• InfillCriteria/ExpectedImprovementBase.cs (added)
• InfillCriteria/ExpectedQuality.cs (modified) (2 diffs)
• InfillCriteria/ExpectedQuantileImprovement.cs (modified) (3 diffs)
• InfillCriteria/InfillCriterionBase.cs (modified) (2 diffs)
• InfillCriteria/MinimalQuantileCriterium.cs (modified) (2 diffs)
• InfillCriteria/NeighbourDistance.cs (added)
• InfillCriteria/PluginExpectedImprovement.cs (modified) (3 diffs)
• InfillCriteria/RobustImprovement.cs (deleted)
• Interfaces/IInfillCriterion.cs (modified) (1 diff)
• Operators (added)
• Operators/CorrelationAnalyzer.cs (added)
• Operators/EvaluatedSolutionsAnalyzer.cs (added)
• Operators/FitnessClusteringAnalyzer.cs (added)
• Operators/InfillSolver.cs (added)
• Operators/ModelBuilder.cs (added)
• Operators/ModelQualityAnalyzer.cs (added)
• Operators/SampleCollector.cs (added)
• Operators/VariableVariabilityAnalyzer.cs (added)
• Plugin.cs.frame (modified) (2 diffs)
• Problems/InfillProblem.cs (modified) (4 diffs)
• SamplingMethods/LatinHyperCubeDesignCreator.cs (added)
• SamplingMethods/LatinHyperCubeDesignOLD.cs (added)
• SamplingMethods/UniformRandomSampling.cs (modified) (1 diff)

branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/EfficientGlobalOptimizationAlgorithm.cs

@@ -24 +24 @@
 using System.Linq;
 using System.Threading;
-using System.Windows.Forms;
 using HeuristicLab.Algorithms.DataAnalysis;
 using HeuristicLab.Analysis;
@@ -35 +34 @@
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
-using HeuristicLab.Problems.Instances.DataAnalysis;
-using HeuristicLab.Problems.Instances.DataAnalysis.Views;
+using HeuristicLab.Problems.Instances;
 using HeuristicLab.Random;
 
@@ -42 +40 @@
   [StorableClass]
   [Creatable(CreatableAttribute.Categories.Algorithms, Priority = 95)]
-  [Item("EfficientGlobalOptimizationAlgortihm", "Solves a problem by sequentially learning a model, solving a subproblem on the model and evaluating the best found solution for this subproblem.")]
+  [Item("EfficientGlobalOptimizationAlgorithm", "Solves a problem by sequentially learning a model, solving a subproblem on the model and evaluating the best found solution for this subproblem.")]
   public class EfficientGlobalOptimizationAlgorithm : BasicAlgorithm, ISurrogateAlgorithm<RealVector> {
     #region Basic-Alg-Essentials
@@ -84 +82 @@
     #endregion
 
-    #region TransmissionResultNames
-    public const string BestInfillSolutionResultName = "BestInfillSolution";
-    public const string BestInfillQualityResultName = "BestInfillQuality";
-    #endregion
-
     #region ParameterProperties
     public IFixedValueParameter<IntValue> GenerationSizeParemeter => Parameters[GenerationSizeParameterName] as IFixedValueParameter<IntValue>;
@@ -108 +101 @@
 
     #region Properties
-
     public int GenerationSize => GenerationSizeParemeter.Value.Value;
     public IInfillCriterion InfillCriterion => InfillCriterionParameter.Value;
@@ -123 +115 @@
       ? Samples.Skip(Samples.Count - MaximalDatasetSize)
       : Samples;
-
     private bool RemoveDuplicates => RemoveDuplicatesParameter.Value.Value;
     private RealVector BaselineVector => BaselineVectorParameter.Value;
@@ -161 +152 @@
     private DataTable ResultsQualities => (DataTable)Results[QualitiesChartResultName].Value;
     private DataRow ResultsQualitiesBest => ResultsQualities.Rows[BestQualitiesRowResultName];
-
     private DataRow ResultsQualitiesWorst => ResultsQualities.Rows[WorstQualitiesRowResultName];
-
     private DataRow ResultsQualitiesIteration => ResultsQualities.Rows[CurrentQualitiesRowResultName];
-
     private IRegressionSolution ResultsModel
     {
@@ -177 +165 @@
     protected EfficientGlobalOptimizationAlgorithm(bool deserializing) : base(deserializing) { }
     [StorableHook(HookType.AfterDeserialization)]
-    private void AfterDeseialization() {
+    protected void AfterDeseialization() {
       RegisterEventhandlers();
     }
-    protected EfficientGlobalOptimizationAlgorithm(EfficientGlobalOptimizationAlgorithm original, Cloner cloner)
-      : base(original, cloner) {
+    protected EfficientGlobalOptimizationAlgorithm(EfficientGlobalOptimizationAlgorithm original, Cloner cloner) : base(original, cloner) {
       Random = cloner.Clone(Random);
       if (original.Samples != null) Samples = original.Samples.Select(x => new Tuple<RealVector, double>(cloner.Clone(x.Item1), x.Item2)).ToList();
-      if (original.InitialSamples != null) Samples = original.InitialSamples.Select(x => new Tuple<RealVector, double>(cloner.Clone(x.Item1), x.Item2)).ToList();
+      if (original.InitialSamples != null) InitialSamples = original.InitialSamples.Select(x => new Tuple<RealVector, double>(cloner.Clone(x.Item1), x.Item2)).ToList();
       RegisterEventhandlers();
     }
     public override IDeepCloneable Clone(Cloner cloner) { return new EfficientGlobalOptimizationAlgorithm(this, cloner); }
     public EfficientGlobalOptimizationAlgorithm() {
+      IProblemInstanceExporter dummy = new RegressionProblem(); //this variable is irrelevant
+      //the dummy variable enforces a using-Statement for HeuristicLab.Problems.Instances
+      //"new ValueParameter<IDataAnalysisAlgorithm<IRegressionProblem>>" requires no using using-Statement, but nontheless it requires HeuristicLab.Problems.Instances to be referenced  
+      //Having HeuristicLab.Problems.Instances referenced but not used, causes the Essential-Unit-tests to fail. 
+
       var cmaes = new CMAEvolutionStrategy.CMAEvolutionStrategy {
         MaximumGenerations = 300,
@@ -206 +198 @@
       Parameters.Add(new FixedValueParameter<IntValue>(InfillOptimizationRestartsParameterName, "Number of restarts of the SubAlgortihm to avoid local optima", new IntValue(1)));
       Parameters.Add(new FixedValueParameter<IntValue>(GenerationSizeParameterName, "Number points that are sampled every iteration (stadard EGO: 1)", new IntValue(1)));
-      Parameters.Add(new ConstrainedValueParameter<IInfillCriterion>(InfillCriterionParameterName, "Decision what value should decide the next sample"));
-      InfillCriterionParameter.ValidValues.Add(new ExpectedImprovement());
-      InfillCriterionParameter.ValidValues.Add(new AugmentedExpectedImprovement());
-      InfillCriterionParameter.ValidValues.Add(new ExpectedQuality());
-      var eqi = new ExpectedQuantileImprovement();
-      InfillCriterionParameter.ValidValues.Add(eqi);
-      eqi.MaxEvaluationsParameter.Value = MaximumEvaluationsParameter.Value;
-      InfillCriterionParameter.ValidValues.Add(new MinimalQuantileCriterium());
-      InfillCriterionParameter.ValidValues.Add(new RobustImprovement());
-      InfillCriterionParameter.ValidValues.Add(new PluginExpectedImprovement());
       Parameters.Add(new FixedValueParameter<IntValue>(MaximalDataSetSizeParameterName, "The maximum number of sample points used to generate the model. Set 0 or less to use always all samples ", new IntValue(-1)));
       Parameters.Add(new FixedValueParameter<BoolValue>(RemoveDuplicatesParamterName, "Wether duplicate samples should be replaced by a single sample with an averaged quality. This GREATLY decreases the chance of ill conditioned models (unbuildable models) but is not theoretically sound as the model ignores the increasing certainty in this region"));
       Parameters.Add(new FixedValueParameter<FileValue>(InitialSamplesParameterName, "The file specifying some initial samples used to jump start the algorithm. These samples are not counted as evaluations. If InitialEvaluations is more than the samples specified in the file, the rest is uniformly random generated and evaluated.", new FileValue()));
       Parameters.Add(new ValueParameter<RealVector>(BaselineVectorParameterName, "A vector used to create a baseline, this vector is evaluated once and is not part of the modeling process (has no influence on algorithm performance)"));
-      var intialSamplingPlans = new ItemSet<IInitialSampling> { new UniformRandomSampling(), new LatinHyperCubeDesign() };
-      Parameters.Add(new ConstrainedValueParameter<IInitialSampling>(InitialSamplingPlanParamterName, intialSamplingPlans, intialSamplingPlans.First()));
-
+      var eqi = new ExpectedQuantileImprovement();
+      eqi.MaxEvaluationsParameter.Value = MaximumEvaluationsParameter.Value;
+      var criteria = new ItemSet<IInfillCriterion> { new ExpectedImprovement(), new AugmentedExpectedImprovement(), new ExpectedQuality(), eqi, new MinimalQuantileCriterium(), new PluginExpectedImprovement() };
+      Parameters.Add(new ConstrainedValueParameter<IInfillCriterion>(InfillCriterionParameterName, "Decision what value should decide the next sample", criteria, criteria.First()));
+      var intialSamplingPlans = new ItemSet<IInitialSampling> { new UniformRandomSampling(), new LatinHyperCubeDesignCreator() };
+      Parameters.Add(new ConstrainedValueParameter<IInitialSampling>(InitialSamplingPlanParamterName, "Determies the initial samples from which the first model can be built.", intialSamplingPlans, intialSamplingPlans.First()));
       SetInfillProblem();
       RegisterEventhandlers();
     }
     #endregion
-
+    public void SetInitialSamples(RealVector[] individuals, double[] qualities) {
+      InitialSamples = individuals.Zip(qualities, (individual, d) => new Tuple<RealVector, double>(individual, d)).ToList();
+    }
     protected override void Initialize(CancellationToken cancellationToken) {
       base.Initialize(cancellationToken);
@@ -235 +222 @@
       var infillProblem = InfillOptimizationAlgorithm.Problem as InfillProblem;
       if (infillProblem == null) throw new ArgumentException("InfillOptimizationAlgorithm has no InfillProblem. Troubles with Eventhandling?");
-      infillProblem.Problem = Problem;
-
 
       //random
@@ -259 +244 @@
 
     }
-
     protected override void Run(CancellationToken cancellationToken) {
       //initial samples
@@ -294 +278 @@
     }
 
-    public void SetInitialSamples(RealVector[] individuals, double[] qualities) {
-      InitialSamples = individuals.Zip(qualities, (individual, d) => new Tuple<RealVector, double>(individual, d)).ToList();
-    }
-
     #region Eventhandling
     private void RegisterEventhandlers() {
@@ -318 +298 @@
     private void OnInfillOptimizationAlgorithmChanged(object sender, EventArgs args) {
       SetInfillProblem();
-      InfillOptimizationAlgorithm.ProblemChanged -= InfillOptimizationProblemChanged;
       InfillOptimizationAlgorithm.ProblemChanged += InfillOptimizationProblemChanged;
     }
@@ -334 +313 @@
       RegressionAlgorithm.Problem = new RegressionProblem();
     }
-    private void OnInitialSamplesChanged(object sender, EventArgs args) {
-      IRegressionProblemData samplesData = null;
-      using (var importTypeDialog = new RegressionImportTypeDialog()) {
-        if (importTypeDialog.ShowDialog() != DialogResult.OK) return;
-        samplesData = new RegressionCSVInstanceProvider().ImportData(importTypeDialog.Path, importTypeDialog.ImportType, importTypeDialog.CSVFormat);
-        InitialSamplesParameter.ToStringChanged -= OnInitialSamplesChanged;
-        InitialSamplesParameter.Value.Value = importTypeDialog.Path;
-        InitialSamplesParameter.ToStringChanged -= OnInitialSamplesChanged;
-
-      }
-
-
-
-      var solutions = new RealVector[samplesData.Dataset.Rows];
-      var qualities = new double[samplesData.Dataset.Rows];
-      var inputVariables = samplesData.InputVariables.CheckedItems.ToArray();
-      for (var i = 0; i < solutions.Length; i++) {
-        qualities[i] = samplesData.Dataset.GetDoubleValue(samplesData.TargetVariable, i);
-        solutions[i] = new RealVector(inputVariables.Length);
-        for (var j = 0; j < inputVariables.Length; j++) solutions[i][j] = samplesData.Dataset.GetDoubleValue(inputVariables[j].Value.Value, i);
-      }
-
-      SetInitialSamples(solutions, qualities);
-
-    }
-
+    private void OnInitialSamplesChanged(object sender, EventArgs args) { }
     protected override void OnExecutionTimeChanged() {
       base.OnExecutionTimeChanged();
@@ -376 +330 @@
       base.Stop();
     }
-    protected override void OnProblemChanged() {
-      base.OnProblemChanged();
-      var infillProblem = InfillOptimizationAlgorithm.Problem as InfillProblem;
-      if (infillProblem == null) throw new ArgumentException("InfillOptimizationAlgorithm has no InfillProblem. Troubles with Eventhandling?");
-      infillProblem.Problem = Problem;
-    }
     #endregion
 
     #region helpers
-    private void SetInfillProblem() {
-      var infillProblem = new InfillProblem {
-        InfillCriterion = InfillCriterion,
-        Problem = Problem
-      };
-      InfillOptimizationAlgorithm.Problem = infillProblem;
-    }
     private IRegressionSolution BuildModel(CancellationToken cancellationToken) {
       var dataset = EgoUtilities.GetDataSet(DataSamples.ToList(), RemoveDuplicates);
@@ -406 +347 @@
       IRegressionSolution solution = null;
 
-      while (solution == null && i++ < 100) {  //TODO: Question: Why does GP degenerate to NaN so often? Answer: There is not even the slightest mitigation strategy for "almost duplicates" that ill-condition the covariance matrix.
+      while (solution == null && i++ < 100) {
         var results = EgoUtilities.SyncRunSubAlgorithm(RegressionAlgorithm, Random.Next(int.MaxValue));
         solution = results.Select(x => x.Value).OfType<IRegressionSolution>().SingleOrDefault();
@@ -451 +392 @@
       //parameterize and check InfillProblem
       var infillProblem = InfillOptimizationAlgorithm.Problem as InfillProblem;
-      if (infillProblem == null) throw new ArgumentException("InfillOptimizationAlgorithm does not have InfillProblem. Problem with Eventhandling?");
-      if (infillProblem.InfillCriterion != InfillCriterion) throw new ArgumentException("InfillCiriterion for Problem is not correct. Problem with Eventhandling?");
-      if (infillProblem.Problem != Problem) throw new ArgumentException("Expensive real problem is not correctly set in InfillProblem. Problem with Eventhandling?");
-      InfillCriterion.Initialize(ResultsModel, Problem.Maximization, infillProblem.Encoding);
+      if (infillProblem == null) throw new ArgumentException("InfillOptimizationAlgorithm does not have an InfillProblem.");
+      if (infillProblem.InfillCriterion != InfillCriterion) throw new ArgumentException("InfillCiriterion for Problem is not correctly set.");
+      var enc = Problem.Encoding as RealVectorEncoding;
+      infillProblem.Encoding.Bounds = enc.Bounds;
+      infillProblem.Encoding.Length = enc.Length;
+      infillProblem.Initialize(ResultsModel, Problem.Maximization);
+
+
 
       RealVector bestVector = null;
       var bestValue = infillProblem.Maximization ? double.NegativeInfinity : double.PositiveInfinity;
-
       for (var i = 0; i < InfillOptimizationRestarts; i++) {
         //optimize
@@ -464 +408 @@
         cancellationToken.ThrowIfCancellationRequested();
         //extract results
-        if (!res.ContainsKey(BestInfillSolutionResultName)) throw new ArgumentException("The InfillOptimizationAlgorithm did not return a best solution");
-        var v = res[BestInfillSolutionResultName].Value as RealVector;
-        if (!res.ContainsKey(BestInfillQualityResultName)) throw new ArgumentException("The InfillOptimizationAlgorithm did not return a best quality");
-        var d = res[BestInfillQualityResultName].Value as DoubleValue;
+        if (!res.ContainsKey(InfillProblem.BestInfillSolutionResultName)) throw new ArgumentException("The InfillOptimizationAlgorithm did not return a best solution");
+        var v = res[InfillProblem.BestInfillSolutionResultName].Value as RealVector;
+        if (!res.ContainsKey(InfillProblem.BestInfillQualityResultName)) throw new ArgumentException("The InfillOptimizationAlgorithm did not return a best quality");
+        var d = res[InfillProblem.BestInfillQualityResultName].Value as DoubleValue;
         if (d == null || v == null) throw new ArgumentException("The InfillOptimizationAlgorithm did not return the expected result types");
-
         //check for improvement
         if (infillProblem.Maximization != d.Value > bestValue) continue;
@@ -475 +418 @@
         bestVector = v;
       }
-
       InfillOptimizationAlgorithm.Runs.Clear();
       return bestVector;
     }
-    private Tuple<RealVector, double> Evaluate(RealVector point) {
-      return new Tuple<RealVector, double>(point, Problem.Evaluate(GetIndividual(point), Random));
-    }
+
     private void Analyze() {
       ResultsEvaluations = Samples.Count;
@@ -493 +433 @@
       ResultsQualitiesWorst.Values.Add(Samples[Problem.Maximization ? min : max].Item2);
       Problem.Analyze(Samples.Select(x => GetIndividual(x.Item1)).ToArray(), Samples.Select(x => x.Item2).ToArray(), Results, Random);
-
-      if (Samples.Count != 0 && Samples[0].Item1.Length == 2) {
-        var plotname = "DEBUG:Sample Distribution";
-        var rowInit = "Initial Samples";
-        var rowAll = "All Samples";
-        if (!Results.ContainsKey(plotname)) Results.Add(new Result(plotname, new ScatterPlot()));
-        var plot = Results[plotname].Value as ScatterPlot;
-        if (!plot.Rows.ContainsKey(rowInit) && InitialSamples != null && InitialSamples.Count > 0)
-          plot.Rows.Add(new ScatterPlotDataRow(rowInit, "samples from inital file (already evaulated)", InitialSamples.Select(x => new Point2D<double>(x.Item1[0], x.Item1[1]))));
-        if (!plot.Rows.ContainsKey(rowAll)) plot.Rows.Add(new ScatterPlotDataRow(rowAll, "All samples", new Point2D<double>[0]));
-        else { plot.Rows[rowAll].Points.Clear(); }
-        plot.Rows[rowAll].Points.AddRange(Samples.Select(x => new Point2D<double>(x.Item1[0], x.Item1[1])));
-
-
-      }
-    }
+      if (Samples.Count != 0 && Samples[0].Item1.Length == 2) AnalyzeSampleDistribution();
+      AnalyzePredictionCorrelation();
+    }
+
+    private void AnalyzeSampleDistribution() {
+      const string plotname = "DEBUG:Sample Distribution";
+      const string rowInit = "Initial Samples";
+      const string rowAll = "All Samples";
+      if (!Results.ContainsKey(plotname)) Results.Add(new Result(plotname, new ScatterPlot()));
+      var plot = (ScatterPlot)Results[plotname].Value;
+      if (!plot.Rows.ContainsKey(rowInit) && InitialSamples != null && InitialSamples.Count > 0)
+        plot.Rows.Add(new ScatterPlotDataRow(rowInit, "samples from inital file (already evaulated)", InitialSamples.Select(x => new Point2D<double>(x.Item1[0], x.Item1[1]))));
+      if (!plot.Rows.ContainsKey(rowAll)) plot.Rows.Add(new ScatterPlotDataRow(rowAll, "All samples", new Point2D<double>[0]));
+      else { plot.Rows[rowAll].Points.Clear(); }
+      plot.Rows[rowAll].Points.AddRange(Samples.Select(x => new Point2D<double>(x.Item1[0], x.Item1[1])));
+    }
+
+    private void AnalyzePredictionCorrelation() {
+      const string plotName = "Prediction";
+      const string rowName = "Samples";
+      const string lastrowName = "Last Sample";
+      if (!Results.ContainsKey(plotName)) Results.Add(new Result(plotName, new ScatterPlot()));
+      var plot = (ScatterPlot)Results[plotName].Value;
+      if (!plot.Rows.ContainsKey(rowName)) plot.Rows.Add(new ScatterPlotDataRow(rowName, rowName, new List<Point2D<double>>()));
+      if (!plot.Rows.ContainsKey(lastrowName)) plot.Rows.Add(new ScatterPlotDataRow(lastrowName, lastrowName, new List<Point2D<double>>()));
+      var p = Samples[Samples.Count - 1];
+      if (ResultsModel != null) plot.Rows[rowName].Points.Add(new Point2D<double>(ResultsModel.Model.GetEstimation(p.Item1), p.Item2, p.Item1));
+      plot.VisualProperties.YAxisTitle = "True Objective Value";
+      plot.VisualProperties.XAxisTitle = "Predicted Objective Value";
+
+    }
+
     private Individual GetIndividual(RealVector r) {
       var scope = new Scope();
@@ -514 +470 @@
       return new SingleEncodingIndividual(Problem.Encoding, scope);
     }
+    private Tuple<RealVector, double> Evaluate(RealVector point) {
+      return new Tuple<RealVector, double>(point, Problem.Evaluate(GetIndividual(point), Random));
+    }
+
+    private void SetInfillProblem() {
+      InfillOptimizationAlgorithm.Problem = new InfillProblem { InfillCriterion = InfillCriterion };
+    }
     #endregion
   }

branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/EgoUtilities.cs

@@ -25 +25 @@
 using System.Threading;
 using HeuristicLab.Common;
+using HeuristicLab.Core;
 using HeuristicLab.Data;
 using HeuristicLab.Encodings.RealVectorEncoding;
@@ -32 +33 @@
 namespace HeuristicLab.Algorithms.EGO {
   internal static class EgoUtilities {
+    //Extention methods for convenience
     public static int ArgMax<T>(this IEnumerable<T> values, Func<T, double> func) {
       var max = double.MinValue;
@@ -49 +51 @@
       return ArgMax(values, x => -func.Invoke(x));
     }
+    public static double GetEstimation(this IRegressionModel model, RealVector r) {
+      var dataset = GetDataSet(new[] { new Tuple<RealVector, double>(r, 0.0) }, false);
+      return model.GetEstimatedValues(dataset, new[] { 0 }).First();
+    }
+    public static double GetVariance(this IConfidenceRegressionModel model, RealVector r) {
+      var dataset = GetDataSet(new[] { new Tuple<RealVector, double>(r, 0.0) }, false);
+      return model.GetEstimatedVariances(dataset, new[] { 0 }).First();
+    }
+    public static double GetDoubleValue(this IDataset dataset, int i, int j) {
+      return dataset.GetDoubleValue("input" + j, i);
+    }
 
+    //Sub-ALgorithms
     public static ResultCollection SyncRunSubAlgorithm(IAlgorithm alg, int random) {
 
@@ -64 +78 @@
       EventWaitHandle trigger = new AutoResetEvent(false);
       Exception ex = null;
-      EventHandler<EventArgs<Exception>> exhandler = (sender, e) => ex = e.Value;
+      EventHandler<EventArgs<Exception>> exhandler = (sender, e) => { ex = e.Value; trigger.Set(); };
       EventHandler stoppedHandler = (sender, e) => trigger.Set();
+
+      alg.ExceptionOccurred -= exhandler; //avoid double attaching in case of pause
       alg.ExceptionOccurred += exhandler;
+      alg.Stopped -= stoppedHandler;
       alg.Stopped += stoppedHandler;
-      alg.Prepare();
+      alg.Paused -= stoppedHandler;
+      alg.Paused += stoppedHandler;
+
+      if (alg.ExecutionState != ExecutionState.Paused) alg.Prepare();
       alg.Start();
       trigger.WaitOne();
@@ -77 +97 @@
     }
 
-    public static double GetEstimation(this IRegressionModel model, RealVector r) {
-      var dataset = GetDataSet(new[] { new Tuple<RealVector, double>(r, 0.0) }, false);
-      return model.GetEstimatedValues(dataset, new[] { 0 }).First();
-    }
-    public static double GetVariance(this IConfidenceRegressionModel model, RealVector r) {
-      var dataset = GetDataSet(new[] { new Tuple<RealVector, double>(r, 0.0) }, false);
-      return model.GetEstimatedVariances(dataset, new[] { 0 }).First();
-    }
-
-
-    public static double GetDoubleValue(this IDataset dataset, int i, int j) {
-      return dataset.GetDoubleValue("input" + j, i);
-    }
+    //RegressionModel extensions
+    public const double DuplicateResolution = 0.0001;
     public static Dataset GetDataSet(IReadOnlyList<Tuple<RealVector, double>> samples, bool removeDuplicates) {
-      if (removeDuplicates)
-        samples = RemoveDuplicates(samples); //TODO duplicates require heteroskedasticity in Models
-
-
+      if (removeDuplicates) samples = RemoveDuplicates(samples); //TODO duplicate removal leads to incorrect uncertainty values in models
       var dimensions = samples[0].Item1.Length + 1;
       var data = new double[samples.Count, dimensions];
       var names = new string[dimensions - 1];
       for (var i = 0; i < names.Length; i++) names[i] = "input" + i;
-
       for (var j = 0; j < samples.Count; j++) {
         for (var i = 0; i < names.Length; i++) data[j, i] = samples[j].Item1[i];
         data[j, dimensions - 1] = samples[j].Item2;
-
       }
-
-
       return new Dataset(names.Concat(new[] { "output" }).ToArray(), data);
     }
-
     private static IReadOnlyList<Tuple<RealVector, double>> RemoveDuplicates(IReadOnlyList<Tuple<RealVector, double>> samples) {
       var res = new List<Tuple<RealVector, double, int>>();
-
       foreach (var sample in samples) {
         if (res.Count == 0) {
@@ -118 +118 @@
           continue;
         }
-
         var index = res.ArgMin(x => Euclidian(sample.Item1, x.Item1));
         var d = Euclidian(res[index].Item1, sample.Item1);
-        if (d > 0.0001)
+        if (d > DuplicateResolution)
           res.Add(new Tuple<RealVector, double, int>(sample.Item1, sample.Item2, 1));
         else {
@@ -131 +130 @@
       return res.Select(x => new Tuple<RealVector, double>(x.Item1, x.Item2 / x.Item3)).ToArray();
     }
-
     private static double Euclidian(IEnumerable<double> a, IEnumerable<double> b) {
       return Math.Sqrt(a.Zip(b, (d, d1) => d - d1).Sum(d => d * d));
     }
-
-    public static DoubleMatrix GetBoundingBox(IEnumerable<RealVector> vectors) {
-      DoubleMatrix res = null;
-      foreach (var vector in vectors)
-        if (res == null) {
-          res = new DoubleMatrix(vector.Length, 2);
-          for (var i = 0; i < vector.Length; i++)
-            res[i, 0] = res[i, 1] = vector[i];
-        } else
-          for (var i = 0; i < vector.Length; i++) {
-            res[i, 0] = Math.Min(vector[i], res[i, 0]);
-            res[i, 1] = Math.Max(vector[i], res[i, 1]);
-          }
-      return res;
-    }
-
-
   }
 }

branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/HeuristicLab.Algorithms.EGO-3.4.csproj

@@ -39 +39 @@
       <HintPath>..\..\..\trunk\sources\bin\HeuristicLab.Algorithms.DataAnalysis-3.4.dll</HintPath>
     </Reference>
-    <Reference Include="HeuristicLab.Algorithms.EvolutionStrategy-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
-      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\trunk\sources\bin\HeuristicLab.Algorithms.EvolutionStrategy-3.3.dll</HintPath>
-    </Reference>
     <Reference Include="HeuristicLab.Analysis-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
@@ -55 +51 @@
       <HintPath>..\..\..\trunk\sources\bin\HeuristicLab.Common-3.3.dll</HintPath>
     </Reference>
-    <Reference Include="HeuristicLab.Common.Resources-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
-      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\trunk\sources\bin\HeuristicLab.Common.Resources-3.3.dll</HintPath>
-    </Reference>
     <Reference Include="HeuristicLab.Core-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
@@ -66 +58 @@
       <SpecificVersion>False</SpecificVersion>
       <HintPath>..\..\..\trunk\sources\bin\HeuristicLab.Data-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Encodings.PermutationEncoding-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\..\trunk\sources\bin\HeuristicLab.Encodings.PermutationEncoding-3.3.dll</HintPath>
     </Reference>
     <Reference Include="HeuristicLab.Encodings.RealVectorEncoding-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
@@ -75 +71 @@
       <SpecificVersion>False</SpecificVersion>
       <HintPath>..\..\..\trunk\sources\bin\HeuristicLab.Optimization-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Optimization.Operators-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\..\trunk\sources\bin\HeuristicLab.Optimization.Operators-3.3.dll</HintPath>
     </Reference>
     <Reference Include="HeuristicLab.Parameters-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
@@ -92 +92 @@
       <HintPath>..\..\..\trunk\sources\bin\HeuristicLab.Problems.DataAnalysis-3.4.dll</HintPath>
     </Reference>
-    <Reference Include="HeuristicLab.Problems.Instances-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+    <Reference Include="HeuristicLab.Problems.Instances-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec" />
+    <Reference Include="HeuristicLab.Problems.SurrogateProblem-3.3, Version=3.3.0.0, Culture=neutral, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\trunk\sources\bin\HeuristicLab.Problems.Instances-3.3.dll</HintPath>
-    </Reference>
-    <Reference Include="HeuristicLab.Problems.Instances.DataAnalysis-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
-      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\trunk\sources\bin\HeuristicLab.Problems.Instances.DataAnalysis-3.3.dll</HintPath>
-    </Reference>
-    <Reference Include="HeuristicLab.Problems.Instances.DataAnalysis.Views-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
-      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\trunk\sources\bin\HeuristicLab.Problems.Instances.DataAnalysis.Views-3.3.dll</HintPath>
+      <HintPath>..\..\..\trunk\sources\bin\HeuristicLab.Problems.SurrogateProblem-3.3.dll</HintPath>
     </Reference>
     <Reference Include="HeuristicLab.Random-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
@@ -108 +101 @@
       <HintPath>..\..\..\trunk\sources\bin\HeuristicLab.Random-3.3.dll</HintPath>
     </Reference>
+    <Reference Include="Microsoft.Build.Tasks.v4.0" />
     <Reference Include="System" />
     <Reference Include="System.Core" />
-    <Reference Include="System.Windows.Forms" />
-    <Reference Include="System.Xml.Linq" />
-    <Reference Include="System.Data.DataSetExtensions" />
     <Reference Include="Microsoft.CSharp" />
-    <Reference Include="System.Data" />
-    <Reference Include="System.Net.Http" />
-    <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
@@ -122 +110 @@
     <Compile Include="EfficientGlobalOptimizationAlgorithm.cs" />
     <Compile Include="InfillCriteria\AugmentedExpectedImprovement.cs" />
+    <Compile Include="InfillCriteria\ExpectedImprovementBase.cs" />
+    <Compile Include="InfillCriteria\NeighbourDistance.cs" />
     <Compile Include="InfillCriteria\ExpectedQuantileImprovement.cs" />
     <Compile Include="InfillCriteria\PluginExpectedImprovement.cs" />
-    <Compile Include="InfillCriteria\RobustImprovement.cs" />
     <Compile Include="InfillCriteria\MinimalQuantileCriterium.cs" />
     <Compile Include="InfillCriteria\ExpectedQuality.cs" />
@@ -132 +121 @@
     <Compile Include="Interfaces\ISurrogateAlgorithm.cs" />
     <Compile Include="Interfaces\IInfillCriterion.cs" />
+    <Compile Include="AdaptiveSamplingAlgorithm.cs" />
+    <Compile Include="Operators\FitnessClusteringAnalyzer.cs" />
+    <Compile Include="Operators\VariableVariabilityAnalyzer.cs" />
+    <Compile Include="Operators\ModelQualityAnalyzer.cs" />
+    <Compile Include="Operators\EvaluatedSolutionsAnalyzer.cs" />
+    <Compile Include="Operators\ModelBuilder.cs" />
+    <Compile Include="Operators\InfillSolver.cs" />
+    <Compile Include="Operators\CorrelationAnalyzer.cs" />
+    <Compile Include="Operators\SampleCollector.cs" />
     <Compile Include="Plugin.cs" />
     <Compile Include="Problems\InfillProblem.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
-    <Compile Include="SamplingMethods\LatinHyperCubeDesign.cs" />
+    <Compile Include="SamplingMethods\LatinHyperCubeDesignOLD.cs" />
+    <Compile Include="SamplingMethods\LatinHyperCubeDesignCreator.cs" />
     <Compile Include="SamplingMethods\UniformRandomSampling.cs" />
   </ItemGroup>
@@ -142 +141 @@
     <None Include="Properties\AssemblyInfo.cs.frame" />
   </ItemGroup>
-  <ItemGroup>
-    <WCFMetadata Include="Service References\" />
-  </ItemGroup>
+  <ItemGroup />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <PropertyGroup>

branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/AugmentedExpectedImprovement.cs

@@ -35 +35 @@
   [StorableClass]
   [Item("AugmentedExpectedImprovement", "Noisy InfillCriterion, Extension of the Expected Improvement as described in\n Global optimization of stochastic black-box systems via sequential kriging meta-models.\r\nHuang, D., Allen, T., Notz, W., Zeng, N.")]
-  public class AugmentedExpectedImprovement : ExpectedImprovement {
-
-
+  public class AugmentedExpectedImprovement : ExpectedImprovementBase {
 
     #region Parameternames
-
     public const string AlphaParameterName = "Alpha";
-
     #endregion
 
     #region Parameters
-
     public IValueParameter<DoubleValue> AlphaParameter => Parameters[AlphaParameterName] as IValueParameter<DoubleValue>;
-
     #endregion
 
     #region Properties
-
     public double Alpha => AlphaParameter.Value.Value;
     [Storable]
     private double Tau;
-
     #endregion
 
-
-    #region HL-Constructors, Serialization and Cloning 
+    #region Constructors, Serialization and Cloning 
     [StorableConstructor]
-    private AugmentedExpectedImprovement(bool deserializing) : base(deserializing) { }
-
-    private AugmentedExpectedImprovement(AugmentedExpectedImprovement original, Cloner cloner) : base(original, cloner) {
+    protected AugmentedExpectedImprovement(bool deserializing) : base(deserializing) { }
+    protected AugmentedExpectedImprovement(AugmentedExpectedImprovement original, Cloner cloner) : base(original, cloner) {
       Tau = original.Tau;
     }
-
     public AugmentedExpectedImprovement() {
       Parameters.Add(new ValueParameter<DoubleValue>(AlphaParameterName, "The Alpha value specifiying the robustness of the \"effective best solution\". Recommended value is 1", new DoubleValue(1.0)));
-
     }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -82 +70 @@
     }
 
-    protected override void Initialize() {
-      if (ExpensiveMaximization) throw new NotImplementedException("AugmentedExpectedImprovement for maximization not yet implemented");
-      var solution = RegressionSolution as IConfidenceRegressionSolution;
-      if (solution == null) throw new ArgumentException("can not calculate Augmented EI without a regression solution providing confidence values");
+    protected override double Evaluate(RealVector vector, double estimatedFitness, double estimatedStandardDeviation) {
+      var d = GetEstimatedImprovement(BestFitness, estimatedFitness, estimatedStandardDeviation, ExploitationWeight, ExpensiveMaximization);
+      return d * (1 - Tau / Math.Sqrt(estimatedStandardDeviation * estimatedStandardDeviation + Tau * Tau));
+    }
 
+    protected override double FindBestFitness(IConfidenceRegressionSolution solution) {
       Tau = RegressionSolution.EstimatedTrainingValues.Zip(RegressionSolution.ProblemData.TargetVariableTrainingValues, (d, d1) => Math.Abs(d - d1)).Average();
-      var xss = new RealVector(Encoding.Length);
+      var bestSolution = new RealVector(Encoding.Length);
       var xssIndex = solution.EstimatedTrainingValues.Zip(solution.EstimatedTrainingValues, (m, s2) => m + Alpha * Math.Sqrt(s2)).ArgMin(x => x);
       var i = solution.ProblemData.TrainingIndices.ToArray()[xssIndex];
-      for (var j = 0; j < Encoding.Length; j++) xss[j] = solution.ProblemData.Dataset.GetDoubleValue(i, j);
-
-      YMin = RegressionSolution.Model.GetEstimation(xss);
+      for (var j = 0; j < Encoding.Length; j++) bestSolution[j] = solution.ProblemData.Dataset.GetDoubleValue(i, j);
+      return RegressionSolution.Model.GetEstimation(bestSolution);
     }
   }

branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/ExpectedImprovement.cs

@@ -24 +24 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
-using HeuristicLab.Data;
 using HeuristicLab.Encodings.RealVectorEncoding;
-using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
@@ -35 +33 @@
   [StorableClass]
   [Item("ExpectedImprovementMeassure", "Extension of the Expected Improvement to a weighted version by ANDRAS SÓBESTER , STEPHEN J. LEARY and ANDY J. KEANE   in \n On the Design of Optimization Strategies Based on Global Response Surface Approximation Models")]
-  public class ExpectedImprovement : InfillCriterionBase {
-
-    #region ParameterNames
-    private const string ExploitationWeightParameterName = "ExploitationWeight";
-    #endregion
-
-    #region ParameterProperties
-    public IFixedValueParameter<DoubleValue> ExploitationWeightParameter => Parameters[ExploitationWeightParameterName] as IFixedValueParameter<DoubleValue>;
-
-    #endregion
-
-    #region Properties
-    protected double ExploitationWeight => ExploitationWeightParameter.Value.Value;
-
-    [Storable]
-    protected double YMin;
-    #endregion
-
-    #region HL-Constructors, Serialization and Cloning 
+  public sealed class ExpectedImprovement : ExpectedImprovementBase {
+    #region Constructors, Serialization and Cloning 
     [StorableConstructor]
-    protected ExpectedImprovement(bool deserializing) : base(deserializing) { }
-    [StorableHook(HookType.AfterDeserialization)]
-    private void AfterDeserialization() {
-      RegisterEventhandlers();
-    }
-    protected ExpectedImprovement(ExpectedImprovement original, Cloner cloner) : base(original, cloner) {
-      RegisterEventhandlers();
-    }
-    public ExpectedImprovement() {
-      Parameters.Add(new FixedValueParameter<DoubleValue>(ExploitationWeightParameterName, "A value between 0 and 1 indicating the focus on exploration (0) or exploitation (1)", new DoubleValue(0.5)));
-      RegisterEventhandlers();
-    }
+    private ExpectedImprovement(bool deserializing) : base(deserializing) { }
+    private ExpectedImprovement(ExpectedImprovement original, Cloner cloner) : base(original, cloner) { }
+    public ExpectedImprovement() { }
     public override IDeepCloneable Clone(Cloner cloner) {
       return new ExpectedImprovement(this, cloner);
@@ -76 +48 @@
       var yhat = model.GetEstimation(vector);
       var s = Math.Sqrt(model.GetVariance(vector));
-      return GetEstimatedImprovement(YMin, yhat, s, ExploitationWeight);
+      return GetEstimatedImprovement(BestFitness, yhat, s, ExploitationWeight, ExpensiveMaximization);
     }
 
-    public override bool Maximization() {
-      return true;
+    protected override double Evaluate(RealVector vector, double estimatedFitness, double estimatedStandardDeviation) {
+      return GetEstimatedImprovement(BestFitness, estimatedFitness, estimatedStandardDeviation, ExploitationWeight, ExpensiveMaximization);
     }
 
-    protected override void Initialize() {
-      if (ExpensiveMaximization) throw new NotImplementedException("Expected Improvement for maximization not yet implemented");
-      var model = RegressionSolution.Model as IConfidenceRegressionModel;
-      if (model == null) throw new ArgumentException("can not calculate EI without confidence measure");
-      YMin = RegressionSolution.ProblemData.TargetVariableTrainingValues.Min();
+    protected override double FindBestFitness(IConfidenceRegressionSolution solution) {
+      return ExpensiveMaximization ? solution.ProblemData.TargetVariableTrainingValues.Max() : solution.ProblemData.TargetVariableTrainingValues.Min();
     }
-
-    #region Eventhandling
-    private void RegisterEventhandlers() {
-      DeregisterEventhandlers();
-      ExploitationWeightParameter.Value.ValueChanged += OnExploitationWeightChanged;
-    }
-    private void DeregisterEventhandlers() {
-      ExploitationWeightParameter.Value.ValueChanged -= OnExploitationWeightChanged;
-    }
-    private void OnExploitationWeightChanged(object sender, EventArgs e) {
-      ExploitationWeightParameter.Value.ValueChanged -= OnExploitationWeightChanged;
-      ExploitationWeightParameter.Value.Value = Math.Max(0, Math.Min(ExploitationWeight, 1));
-      ExploitationWeightParameter.Value.ValueChanged += OnExploitationWeightChanged;
-    }
-    #endregion
-
-    #region Helpers
-    protected static double GetEstimatedImprovement(double ymin, double yhat, double s, double w) {
-      if (Math.Abs(s) < double.Epsilon) return 0;
-      var val = (ymin - yhat) / s;
-      var res = w * (ymin - yhat) * StandardNormalDistribution(val) + (1 - w) * s * StandardNormalDensity(val);
-      return double.IsInfinity(res) || double.IsNaN(res) ? 0 : res;
-    }
-
-    private static double StandardNormalDensity(double x) {
-      if (Math.Abs(x) > 10) return 0;
-      return Math.Exp(-0.5 * x * x) / Math.Sqrt(2 * Math.PI);
-    }
-
-    //taken from https://www.johndcook.com/blog/2009/01/19/stand-alone-error-function-erf/
-    private static double StandardNormalDistribution(double x) {
-      if (x > 10) return 1;
-      if (x < -10) return 0;
-      const double a1 = 0.254829592;
-      const double a2 = -0.284496736;
-      const double a3 = 1.421413741;
-      const double a4 = -1.453152027;
-      const double a5 = 1.061405429;
-      const double p = 0.3275911;
-      var sign = x < 0 ? -1 : 1;
-      x = Math.Abs(x) / Math.Sqrt(2.0);
-      var t = 1.0 / (1.0 + p * x);
-      var y = 1.0 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.Exp(-x * x);
-      return 0.5 * (1.0 + sign * y);
-    }
-    #endregion
   }
 }

branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/ExpectedQuality.cs

@@ -29 +29 @@
 
   [StorableClass]
-  [Item("ExpectedQualityMeassure", "Use simply the qualitypredicted by the model")]
+  [Item("ExpectedQualityMeassure", "Use the quality predicted by the model")]
   public class ExpectedQuality : InfillCriterionBase {
-    #region HL-Constructors, Serialization and Cloning 
+    #region Constructors, Serialization and Cloning 
     [StorableConstructor]
-    private ExpectedQuality(bool deserializing) : base(deserializing) { }
-    private ExpectedQuality(ExpectedQuality original, Cloner cloner) : base(original, cloner) { }
+    protected ExpectedQuality(bool deserializing) : base(deserializing) { }
+    protected ExpectedQuality(ExpectedQuality original, Cloner cloner) : base(original, cloner) { }
     public ExpectedQuality() { }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -42 +42 @@
 
     public override double Evaluate(RealVector vector) {
-      return RegressionSolution.Model.GetEstimation(vector);
+      return ExpensiveMaximization ? RegressionSolution.Model.GetEstimation(vector) : -RegressionSolution.Model.GetEstimation(vector);
     }
 
-    public override bool Maximization() {
-      return ExpensiveMaximization;
-    }
-
-    protected override void Initialize() {
+    public override void Initialize() {
     }
   }

branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/ExpectedQuantileImprovement.cs

@@ -35 +35 @@
   [StorableClass]
   [Item("ExpectedQuantileImprovement", "Noisy InfillCriterion, Extension of the Expected Improvement as described in \n Noisy expectedimprovement and on - line computation time allocation for the optimization of simulators with tunable fidelitys\r\nPicheny, V., Ginsbourger, D., Richet, Y")]
-  public class ExpectedQuantileImprovement : ExpectedImprovement {
+  public class ExpectedQuantileImprovement : ExpectedImprovementBase {
 
     #region Parameternames
@@ -48 +48 @@
 
     #region Properties
-
     public int MaxEvaluations => MaxEvaluationsParameter.Value.Value;
     public double Alpha => AlphaParameter.Value.Value;
     [Storable]
     private double Tau;
-
     #endregion
 
     #region HL-Constructors, Serialization and Cloning 
     [StorableConstructor]
-    private ExpectedQuantileImprovement(bool deserializing) : base(deserializing) { }
-
-    private ExpectedQuantileImprovement(ExpectedQuantileImprovement original, Cloner cloner) : base(original, cloner) {
+    protected ExpectedQuantileImprovement(bool deserializing) : base(deserializing) { }
+    protected ExpectedQuantileImprovement(ExpectedQuantileImprovement original, Cloner cloner) : base(original, cloner) {
       Tau = original.Tau;
     }
-
     public ExpectedQuantileImprovement() {
       Parameters.Add(new FixedValueParameter<DoubleValue>(AlphaParameterName, "The Alpha value specifiying the robustness of the \"effective best solution\". Recommended value is 1.0", new DoubleValue(1.0)));
-      Parameters.Add(new ValueParameter<IntValue>(MaxEvaluationsParameterName, "The maximum number of evaluations allowed for EGO", new IntValue(100)));
+      Parameters.Add(new ValueParameter<IntValue>(MaxEvaluationsParameterName, "The maximum number of evaluations allowed for EGO", new IntValue(500)));
       MaxEvaluationsParameter.Hidden = true;
     }
@@ -74 +70 @@
     #endregion
 
-    public override double Evaluate(RealVector vector) {
-      var model = RegressionSolution.Model as IConfidenceRegressionModel;
-      var s2 = model.GetVariance(vector);
+    protected override double FindBestFitness(IConfidenceRegressionSolution solution) {
+      Tau = RegressionSolution.EstimatedTrainingValues.Zip(solution.ProblemData.TargetVariableTrainingValues, (d, d1) => Math.Abs(d - d1)).Average();
+      Tau = Tau * Tau / (MaxEvaluations - solution.ProblemData.Dataset.Rows % MaxEvaluations + 1);
 
-      var yhat = model.GetEstimation(vector) + Alpha * Math.Sqrt(Tau * s2 / (Tau + s2));
-      var s = Math.Sqrt(s2 * s2 / (Tau + s2));
+      var index = solution.EstimatedTrainingValues.Zip(solution.EstimatedTrainingVariances, (m, s2) => m + Alpha * Math.Sqrt(s2)).ArgMin(x => x);
+      return solution.EstimatedTrainingValues.ToArray()[index];
 
-      return GetEstimatedImprovement(YMin, yhat, s, ExploitationWeight);
     }
 
-    protected override void Initialize() {
-      if (ExpensiveMaximization) throw new NotImplementedException("AugmentedExpectedImprovement for maximization not yet implemented");
-      var solution = RegressionSolution as IConfidenceRegressionSolution;
-      if (solution == null) throw new ArgumentException("can not calculate Augmented EI without a regression solution providing confidence values");
-
-      Tau = RegressionSolution.EstimatedTrainingValues.Zip(RegressionSolution.ProblemData.TargetVariableTrainingValues, (d, d1) => Math.Abs(d - d1)).Average();
-      Tau = Tau * Tau / (MaxEvaluations - RegressionSolution.ProblemData.Dataset.Rows + 1);
-
-      var xss = new RealVector(Encoding.Length);
-      var xssIndex = solution.EstimatedTrainingVariances.Zip(solution.EstimatedTrainingVariances, (m, s2) => m + Alpha * Math.Sqrt(s2)).ArgMin(x => x);
-      var i = solution.ProblemData.TrainingIndices.ToArray()[xssIndex];
-      for (var j = 0; j < Encoding.Length; j++) xss[j] = solution.ProblemData.Dataset.GetDoubleValue(i, j);
-
-      YMin = RegressionSolution.Model.GetEstimation(xss);
+    protected override double Evaluate(RealVector vector, double estimatedFitness, double estimatedStandardDeviation) {
+      var s2 = estimatedStandardDeviation * estimatedStandardDeviation;
+      var penalty = Alpha * Math.Sqrt(Tau * s2 / (Tau + s2));
+      var yhat = estimatedFitness + (ExpensiveMaximization ? -penalty : penalty);
+      var s = Math.Sqrt(s2 * s2 / (Tau + s2));
+      return GetEstimatedImprovement(BestFitness, yhat, s, ExploitationWeight, ExpensiveMaximization);
     }
 

branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/InfillCriterionBase.cs

@@ -30 +30 @@
   [StorableClass]
   public abstract class InfillCriterionBase : ParameterizedNamedItem, IInfillCriterion {
-
     [Storable]
-    protected IRegressionSolution RegressionSolution;
+    public IRegressionSolution RegressionSolution { get; set; }
     [Storable]
-    protected bool ExpensiveMaximization;
+    public bool ExpensiveMaximization { get; set; }
     [Storable]
-    protected RealVectorEncoding Encoding;
+    public RealVectorEncoding Encoding { get; set; }
 
     protected InfillCriterionBase(bool deserializing) : base(deserializing) { }
-
     protected InfillCriterionBase(InfillCriterionBase original, Cloner cloner) : base(original, cloner) {
       RegressionSolution = cloner.Clone(original.RegressionSolution);
@@ -48 +46 @@
 
     public abstract double Evaluate(RealVector vector);
-    public abstract bool Maximization();
+    //public abstract bool Maximization();
 
-    public void Initialize(IRegressionSolution solution, bool expensiveMaximization, RealVectorEncoding encoding) {
-      RegressionSolution = solution;
-      ExpensiveMaximization = expensiveMaximization;
-      Encoding = encoding;
-      Initialize();
-    }
-
-    protected abstract void Initialize();
-
+    public abstract void Initialize();
   }
 }

branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/MinimalQuantileCriterium.cs

@@ -42 +42 @@
     #region ParameterProperties
     public IFixedValueParameter<DoubleValue> ConfidenceWeightParameter => Parameters[ConfidenceWeightParameterName] as IFixedValueParameter<DoubleValue>;
-
     #endregion
 
     #region Properties
     private double ConfidenceWeight => ConfidenceWeightParameter.Value.Value;
-
     #endregion
 
-    #region HL-Constructors, Serialization and Cloning 
+    #region Constructors, Serialization and Cloning 
     [StorableConstructor]
-    private MinimalQuantileCriterium(bool deserializing) : base(deserializing) { }
-    private MinimalQuantileCriterium(MinimalQuantileCriterium original, Cloner cloner) : base(original, cloner) { }
+    protected MinimalQuantileCriterium(bool deserializing) : base(deserializing) { }
+    protected MinimalQuantileCriterium(MinimalQuantileCriterium original, Cloner cloner) : base(original, cloner) { }
     public MinimalQuantileCriterium() {
-      Parameters.Add(new FixedValueParameter<DoubleValue>(ConfidenceWeightParameterName, "A value between 0 and 1 indicating the focus on exploration (0) or exploitation (1)", new DoubleValue(0.5)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(ConfidenceWeightParameterName, "A value greater than 0. The larger the value the stronger the emphasis on exploration", new DoubleValue(0.5)));
     }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -66 +64 @@
       var yhat = model.GetEstimation(vector);
       var s = Math.Sqrt(model.GetVariance(vector)) * ConfidenceWeight;
-      return ExpensiveMaximization ? yhat + s : yhat - s;
+      return (ExpensiveMaximization ? yhat : -yhat) + s;
     }
 
-    public override bool Maximization() {
-      return ExpensiveMaximization;
-    }
 
-    protected override void Initialize() {
+    public override void Initialize() {
       var model = RegressionSolution.Model as IConfidenceRegressionModel;
       if (model == null) throw new ArgumentException("can not calculate EI without confidence measure");

branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/PluginExpectedImprovement.cs

@@ -20 +20 @@
 #endregion
 
-using System;
 using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Encodings.RealVectorEncoding;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
@@ -32 +32 @@
   [StorableClass]
   [Item("PluginExpectedImprovement", "Noisy InfillCriterion, Extension of the Expected Improvement by using the minimal prediction on the observed points\n rather than the minimal observed value as described in \n Global optimization based on noisy evaluations: An empirical study of two statistical approaches\r\nEmmanuel Vazqueza, Julien Villemonteixb, Maryan Sidorkiewiczb and Éric Walterc")]
-  public class PluginExpectedImprovement : ExpectedImprovement {
-
+  public class PluginExpectedImprovement : ExpectedImprovementBase {
 
     #region HL-Constructors, Serialization and Cloning 
     [StorableConstructor]
-    private PluginExpectedImprovement(bool deserializing) : base(deserializing) { }
-    private PluginExpectedImprovement(PluginExpectedImprovement original, Cloner cloner) : base(original, cloner) { }
+    protected PluginExpectedImprovement(bool deserializing) : base(deserializing) { }
+    protected PluginExpectedImprovement(PluginExpectedImprovement original, Cloner cloner) : base(original, cloner) { }
     public PluginExpectedImprovement() { }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -45 +44 @@
     #endregion
 
-    protected override void Initialize() {
-      if (ExpensiveMaximization) throw new NotImplementedException("PluginExpectedImprovement for maximization not yet implemented");
-      var model = RegressionSolution.Model as IConfidenceRegressionModel;
-      if (model == null) throw new ArgumentException("can not calculate EI without confidence measure");
-      YMin = RegressionSolution.EstimatedTrainingValues.Min();
+    protected override double FindBestFitness(IConfidenceRegressionSolution solution) {
+      return ExpensiveMaximization ? RegressionSolution.EstimatedTrainingValues.Max() : RegressionSolution.EstimatedTrainingValues.Min();
+    }
+
+    protected override double Evaluate(RealVector vector, double estimatedFitness, double estimatedStandardDeviation) {
+      return GetEstimatedImprovement(BestFitness, estimatedFitness, estimatedStandardDeviation, ExploitationWeight, ExpensiveMaximization);
     }
   }

branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/Interfaces/IInfillCriterion.cs

@@ -27 +27 @@
 namespace HeuristicLab.Algorithms.EGO {
   public interface IInfillCriterion : INamedItem {
+
+    IRegressionSolution RegressionSolution { get; set; }
+    bool ExpensiveMaximization { get; set; }
+    RealVectorEncoding Encoding { get; set; }
     double Evaluate(RealVector vector);
-    bool Maximization();
-    void Initialize(IRegressionSolution solution, bool expensiveMaximization, RealVectorEncoding encoding);
+    //bool Maximization();
+    void Initialize();
   }
 }

branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/Plugin.cs.frame

@@ -28 +28 @@
   [Plugin("HeuristicLab.Algorithms.EGO", "3.4.5.$WCREV$")]
   [PluginFile("HeuristicLab.Algorithms.EGO-3.4.dll", PluginFileType.Assembly)]
-  [PluginFile("displayModelFrame.html", PluginFileType.Data)]
   [PluginDependency("HeuristicLab.Algorithms.CMAEvolutionStrategy", "3.4")]
   [PluginDependency("HeuristicLab.Algorithms.DataAnalysis", "3.4")]
@@ -34 +33 @@
   [PluginDependency("HeuristicLab.Collections", "3.3")]
   [PluginDependency("HeuristicLab.Common", "3.3")]
-  [PluginDependency("HeuristicLab.Common.Resources", "3.3")]
   [PluginDependency("HeuristicLab.Core", "3.3")]
   [PluginDependency("HeuristicLab.Data", "3.3")]
   [PluginDependency("HeuristicLab.Encodings.RealVectorEncoding", "3.3")]
+  [PluginDependency("HeuristicLab.Encodings.PermutationEncoding", "3.3")]
   [PluginDependency("HeuristicLab.Operators","3.3")]
   [PluginDependency("HeuristicLab.Optimization","3.3")]
+  [PluginDependency("HeuristicLab.Optimization.Operators","3.3")]
   [PluginDependency("HeuristicLab.Parameters","3.3")]
   [PluginDependency("HeuristicLab.Persistence","3.3")]

branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/Problems/InfillProblem.cs

@@ -21 +21 @@
 
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -27 +29 @@
 using HeuristicLab.Optimization;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
 
 namespace HeuristicLab.Algorithms.EGO {
@@ -33 +36 @@
   public sealed class InfillProblem : SingleObjectiveBasicProblem<RealVectorEncoding> {
 
-    public override bool Maximization => true;  //This is necessary because algorithms do not expect the maximization to change
+    public override bool Maximization => true;
 
-    #region Properties;
+    #region ProblemResultNames
+    public const string BestInfillSolutionResultName = "BestInfillSolution";
+    public const string BestInfillQualityResultName = "BestInfillQuality";
+    #endregion
+
+    #region Properties
     [Storable]
     private IInfillCriterion infillCriterion;
-    [Storable]
-    private SingleObjectiveBasicProblem<IEncoding> problem;
 
     public IInfillCriterion InfillCriterion
     {
       get { return infillCriterion; }
-      set { infillCriterion = value; }
-    }
-    public SingleObjectiveBasicProblem<IEncoding> Problem
-    {
-      get { return problem; }
       set
       {
-        problem = value;
-        if (problem == null) return;
-        var enc = problem.Encoding as RealVectorEncoding;
-        if (enc == null) throw new ArgumentException("EGO can not be performed on non-RealVectorEncodings");
-        Encoding = enc;
-        SolutionCreator = new UniformRandomRealVectorCreator();//ignore Problem specific Solution Creation
-
+        infillCriterion = value;
+        infillCriterion.Encoding = Encoding;
       }
     }
     #endregion
 
-    #region HLConstructors
+    #region Constructors
     [StorableConstructor]
     private InfillProblem(bool deserializing) : base(deserializing) { }
     private InfillProblem(InfillProblem original, Cloner cloner) : base(original, cloner) {
-      infillCriterion = cloner.Clone(original.InfillCriterion);
-      problem = cloner.Clone(original.Problem);
+      infillCriterion = cloner.Clone(original.infillCriterion);
     }
     public InfillProblem() { }
@@ -74 +69 @@
 
     public override double Evaluate(Individual individual, IRandom r) {
-      var q = InfillCriterion.Evaluate(individual.RealVector());
-      return InfillCriterion.Maximization() ? q : -q;
+      return !InBounds(individual.RealVector(), Encoding.Bounds) ? double.MinValue : InfillCriterion.Evaluate(individual.RealVector());
     }
     public override void Analyze(Individual[] individuals, double[] qualities, ResultCollection results, IRandom random) {
       base.Analyze(individuals, qualities, results, random);
       var best = qualities.ArgMax(x => x);
-      var qnew = InfillCriterion.Maximization() ? qualities[best] : -qualities[best];
-      const string qname = EfficientGlobalOptimizationAlgorithm.BestInfillQualityResultName;
-      const string sname = EfficientGlobalOptimizationAlgorithm.BestInfillSolutionResultName;
-      if (!results.ContainsKey(EfficientGlobalOptimizationAlgorithm.BestInfillQualityResultName)) {
-        results.Add(new Result(sname, (RealVector)individuals[best].RealVector().Clone()));
-        results.Add(new Result(qname, new DoubleValue(qnew)));
+      var newQuality = qualities[best];
+      if (!results.ContainsKey(BestInfillQualityResultName)) {
+        results.Add(new Result(BestInfillSolutionResultName, (RealVector)individuals[best].RealVector().Clone()));
+        results.Add(new Result(BestInfillQualityResultName, new DoubleValue(newQuality)));
         return;
       }
-      var qold = results[qname].Value as DoubleValue;
+      var qold = results[BestInfillQualityResultName].Value as DoubleValue;
       if (qold == null) throw new ArgumentException("Old best quality is not a double value. Conflicting Analyzers?");
-      if (qold.Value >= qnew == InfillCriterion.Maximization()) return;
-      results[sname].Value = (RealVector)individuals[best].RealVector().Clone();
-      qold.Value = qnew;
+      if (qold.Value >= newQuality) return;
+      results[BestInfillSolutionResultName].Value = (RealVector)individuals[best].RealVector().Clone();
+      qold.Value = newQuality;
     }
+    public override IEnumerable<Individual> GetNeighbors(Individual individual, IRandom random) {
+      var bounds = Encoding.Bounds;
+      var michalewiczIteration = 0;
+      while (true) {
+        var neighbour = individual.Copy();
+        var r = neighbour.RealVector();
+        switch (random.Next(5)) {
+          case 0: UniformOnePositionManipulator.Apply(random, r, bounds); break;
+          case 1: UniformOnePositionManipulator.Apply(random, r, bounds); break;//FixedNormalAllPositionsManipulator.Apply(random, r, new RealVector(new[] { 0.1 })); break;
+          case 2: MichalewiczNonUniformAllPositionsManipulator.Apply(random, r, bounds, new IntValue(michalewiczIteration++), new IntValue(10000), new DoubleValue(5.0)); break;
+          case 3: MichalewiczNonUniformOnePositionManipulator.Apply(random, r, bounds, new IntValue(michalewiczIteration++), new IntValue(10000), new DoubleValue(5.0)); break;
+          case 4: BreederGeneticAlgorithmManipulator.Apply(random, r, bounds, new DoubleValue(0.1)); break;
+          default: throw new NotImplementedException();
+        }
+        yield return neighbour;
+        michalewiczIteration %= 10000;
+      }
+    }
+
+    public void Initialize(IRegressionSolution model, bool expensiveMaximization) {
+      infillCriterion.RegressionSolution = model;
+      infillCriterion.ExpensiveMaximization = expensiveMaximization;
+      infillCriterion.Encoding = Encoding;
+      infillCriterion.Initialize();
+    }
+
+    #region helpers
+    private static bool InBounds(RealVector r, DoubleMatrix bounds) {
+      return !r.Where((t, i) => t < bounds[i % bounds.Rows, 0] || t > bounds[i % bounds.Rows, 1]).Any();
+    }
+    #endregion
+
   }
 }

branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/SamplingMethods/UniformRandomSampling.cs

@@ -34 +34 @@
     #region HL-Constructors, Serialization and Cloning 
     [StorableConstructor]
-    private UniformRandomSampling(bool deserializing) : base(deserializing) { }
-    private UniformRandomSampling(UniformRandomSampling original, Cloner cloner) : base(original, cloner) { }
+    protected UniformRandomSampling(bool deserializing) : base(deserializing) { }
+    protected UniformRandomSampling(UniformRandomSampling original, Cloner cloner) : base(original, cloner) { }
     public UniformRandomSampling() {
     }