Index: /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/EfficientGlobalOptimizationAlgorithm.cs
===================================================================
--- /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/EfficientGlobalOptimizationAlgorithm.cs	(revision 14817)
+++ /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/EfficientGlobalOptimizationAlgorithm.cs	(revision 14818)
@@ -24,4 +24,5 @@
 using System.Linq;
 using System.Threading;
+using System.Windows.Forms;
 using HeuristicLab.Algorithms.DataAnalysis;
 using HeuristicLab.Analysis;
@@ -34,4 +35,6 @@
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.Instances.DataAnalysis;
+using HeuristicLab.Problems.Instances.DataAnalysis.Views;
 using HeuristicLab.Random;
 
@@ -57,5 +60,5 @@
     private const string InfillOptimizationRestartsParameterName = "InfillOptimizationRestarts";
     private const string InitialEvaluationsParameterName = "Initial Evaluations";
-    private const string MaximumIterationsParameterName = "Maximum Iterations";
+    private const string MaximumEvaluationsParameterName = "Maximum Evaluations";
     private const string MaximumRuntimeParameterName = "Maximum Runtime";
     private const string RegressionAlgorithmParameterName = "RegressionAlgorithm";
@@ -63,4 +66,7 @@
     private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
     private const string MaximalDataSetSizeParameterName = "MaximalDataSetSize";
+    private const string RemoveDuplicatesParamterName = "RemoveDuplicates";
+    private const string InitialSamplesParameterName = "InitialSamplesFile";
+    private const string BaselineVectorParameterName = "BaselineVector";
     #endregion
 
@@ -88,5 +94,5 @@
     public IFixedValueParameter<IntValue> InfillOptimizationRestartsParemeter => Parameters[InfillOptimizationRestartsParameterName] as IFixedValueParameter<IntValue>;
     public IFixedValueParameter<IntValue> InitialEvaluationsParameter => Parameters[InitialEvaluationsParameterName] as IFixedValueParameter<IntValue>;
-    public IFixedValueParameter<IntValue> MaximumIterationsParameter => Parameters[MaximumIterationsParameterName] as IFixedValueParameter<IntValue>;
+    public IFixedValueParameter<IntValue> MaximumEvaluationsParameter => Parameters[MaximumEvaluationsParameterName] as IFixedValueParameter<IntValue>;
     public IFixedValueParameter<IntValue> MaximumRuntimeParameter => Parameters[MaximumRuntimeParameterName] as IFixedValueParameter<IntValue>;
     public IValueParameter<IDataAnalysisAlgorithm<IRegressionProblem>> RegressionAlgorithmParameter => Parameters[RegressionAlgorithmParameterName] as IValueParameter<IDataAnalysisAlgorithm<IRegressionProblem>>;
@@ -94,63 +100,30 @@
     public IFixedValueParameter<BoolValue> SetSeedRandomlyParameter => Parameters[SetSeedRandomlyParameterName] as IFixedValueParameter<BoolValue>;
     public IFixedValueParameter<IntValue> MaximalDataSetSizeParameter => Parameters[MaximalDataSetSizeParameterName] as IFixedValueParameter<IntValue>;
+    public IFixedValueParameter<BoolValue> RemoveDuplicatesParameter => Parameters[RemoveDuplicatesParamterName] as IFixedValueParameter<BoolValue>;
+
+    public IFixedValueParameter<FileValue> InitialSamplesParameter => Parameters[InitialSamplesParameterName] as IFixedValueParameter<FileValue>;
+
+    public IValueParameter<RealVector> BaselineVectorParameter => Parameters[BaselineVectorParameterName] as IValueParameter<RealVector>;
     #endregion
 
     #region Properties
 
-    public int GenerationSize
-    {
-      get { return GenerationSizeParemeter.Value.Value; }
-    }
-    public IInfillCriterion InfillCriterion
-    {
-      get { return InfillCriterionParameter.Value; }
-    }
-    public Algorithm InfillOptimizationAlgorithm
-    {
-      get { return InfillOptimizationAlgorithmParameter.Value; }
-    }
-    public int InfillOptimizationRestarts
-    {
-      get { return InfillOptimizationRestartsParemeter.Value.Value; }
-    }
-    public int InitialEvaluations
-    {
-      get { return InitialEvaluationsParameter.Value.Value; }
-    }
-    public int MaximumIterations
-    {
-      get { return MaximumIterationsParameter.Value.Value; }
-    }
-    public int MaximumRuntime
-    {
-      get { return MaximumRuntimeParameter.Value.Value; }
-    }
-    public IDataAnalysisAlgorithm<IRegressionProblem> RegressionAlgorithm
-    {
-      get { return RegressionAlgorithmParameter.Value; }
-    }
-    public int Seed
-    {
-      get { return SeedParameter.Value.Value; }
-    }
-    public bool SetSeedRandomly
-    {
-      get { return SetSeedRandomlyParameter.Value.Value; }
-    }
-    public int MaximalDatasetSize
-    {
-      get { return MaximalDataSetSizeParameter.Value.Value; }
-    }
-
-    private IEnumerable<Tuple<RealVector, double>> DataSamples
-    {
-      get
-      {
-        return Samples.Count > MaximalDatasetSize && MaximalDatasetSize > 0
-          ? Samples.Skip(Samples.Count - MaximalDatasetSize)
-          : Samples;
-      }
-    }
-
+    public int GenerationSize => GenerationSizeParemeter.Value.Value;
+    public IInfillCriterion InfillCriterion => InfillCriterionParameter.Value;
+    public Algorithm InfillOptimizationAlgorithm => InfillOptimizationAlgorithmParameter.Value;
+    public int InfillOptimizationRestarts => InfillOptimizationRestartsParemeter.Value.Value;
+    public int InitialEvaluations => InitialEvaluationsParameter.Value.Value;
+    public int MaximumEvaluations => MaximumEvaluationsParameter.Value.Value;
+    public int MaximumRuntime => MaximumRuntimeParameter.Value.Value;
+    public IDataAnalysisAlgorithm<IRegressionProblem> RegressionAlgorithm => RegressionAlgorithmParameter.Value;
+    public int Seed => SeedParameter.Value.Value;
+    public bool SetSeedRandomly => SetSeedRandomlyParameter.Value.Value;
+    public int MaximalDatasetSize => MaximalDataSetSizeParameter.Value.Value;
+    private IEnumerable<Tuple<RealVector, double>> DataSamples => Samples.Count > MaximalDatasetSize && MaximalDatasetSize > 0
+      ? Samples.Skip(Samples.Count - MaximalDatasetSize)
+      : Samples;
+
+    private bool RemoveDuplicates => RemoveDuplicatesParameter.Value.Value;
+    private RealVector BaselineVector => BaselineVectorParameter.Value;
     #endregion
 
@@ -185,20 +158,11 @@
       set { ((IntValue)Results[IterationsResultName].Value).Value = value; }
     }
-    private DataTable ResultsQualities
-    {
-      get { return (DataTable)Results[QualitiesChartResultName].Value; }
-    }
-    private DataRow ResultsQualitiesBest
-    {
-      get { return ResultsQualities.Rows[BestQualitiesRowResultName]; }
-    }
-    private DataRow ResultsQualitiesWorst
-    {
-      get { return ResultsQualities.Rows[WorstQualitiesRowResultName]; }
-    }
-    private DataRow ResultsQualitiesIteration
-    {
-      get { return ResultsQualities.Rows[CurrentQualitiesRowResultName]; }
-    }
+    private DataTable ResultsQualities => (DataTable)Results[QualitiesChartResultName].Value;
+    private DataRow ResultsQualitiesBest => ResultsQualities.Rows[BestQualitiesRowResultName];
+
+    private DataRow ResultsQualitiesWorst => ResultsQualities.Rows[WorstQualitiesRowResultName];
+
+    private DataRow ResultsQualitiesIteration => ResultsQualities.Rows[CurrentQualitiesRowResultName];
+
     private IRegressionSolution ResultsModel
     {
@@ -232,5 +196,5 @@
       };
       model.CovarianceFunctionParameter.Value = new CovarianceRationalQuadraticIso();
-      Parameters.Add(new FixedValueParameter<IntValue>(MaximumIterationsParameterName, "", new IntValue(int.MaxValue)));
+      Parameters.Add(new FixedValueParameter<IntValue>(MaximumEvaluationsParameterName, "", new IntValue(int.MaxValue)));
       Parameters.Add(new FixedValueParameter<IntValue>(InitialEvaluationsParameterName, "", new IntValue(10)));
       Parameters.Add(new FixedValueParameter<IntValue>(MaximumRuntimeParameterName, "The maximum runtime in seconds after which the algorithm stops. Use -1 to specify no limit for the runtime", new IntValue(3600)));
@@ -242,9 +206,17 @@
       Parameters.Add(new FixedValueParameter<IntValue>(GenerationSizeParameterName, "Number points that are sampled every iteration (stadard EGO: 1)", new IntValue(1)));
       Parameters.Add(new ConstrainedValueParameter<IInfillCriterion>(InfillCriterionParameterName, "Decision what value should decide the next sample"));
+      InfillCriterionParameter.ValidValues.Add(new AugmentedExpectedImprovement());
       InfillCriterionParameter.ValidValues.Add(new ExpectedImprovement());
       InfillCriterionParameter.ValidValues.Add(new ExpectedQuality());
-      InfillCriterionParameter.ValidValues.Add(new ConfidenceBound());
+      var eqi = new ExpectedQuantileImprovement();
+      InfillCriterionParameter.ValidValues.Add(eqi);
+      eqi.MaxEvaluationsParameter.Value = MaximumEvaluationsParameter.Value;
+      InfillCriterionParameter.ValidValues.Add(new MinimalQuantileCriterium());
+      InfillCriterionParameter.ValidValues.Add(new RobustImprovement());
+      InfillCriterionParameter.ValidValues.Add(new PluginExpectedImprovement());
       Parameters.Add(new FixedValueParameter<IntValue>(MaximalDataSetSizeParameterName, "The maximum number of sample points used to generate the model. Set 0 or less to use always all samples ", new IntValue(-1)));
-
+      Parameters.Add(new FixedValueParameter<BoolValue>(RemoveDuplicatesParamterName, "Wether duplicate samples should be replaced by a single sample with an averaged quality. This GREATLY decreases the chance of ill conditioned models (unbuildable models) but is not theoretically sound as the model ignores the increasing certainty in this region"));
+      Parameters.Add(new FixedValueParameter<FileValue>(InitialSamplesParameterName, "The file specifying some initial samples used to jump start the algorithm. These samples are not counted as evaluations. If InitialEvaluations is more than the samples specified in the file, the rest is uniformly random generated and evaluated.", new FileValue()));
+      Parameters.Add(new ValueParameter<RealVector>(BaselineVectorParameterName, "A vector used to create a baseline, this vector is evaluated once and is not part of the modeling process (has no influence on algorithm performance)"));
       SetInfillProblem();
       RegisterEventhandlers();
@@ -257,4 +229,8 @@
       var enc = Problem.Encoding as RealVectorEncoding;
       if (enc == null) throw new ArgumentException("The EGO algorithm can only be applied to RealVectorEncodings");
+      var infillProblem = InfillOptimizationAlgorithm.Problem as InfillProblem;
+      if (infillProblem == null) throw new ArgumentException("InfillOptimizationAlgorithm has no InfillProblem. Troubles with Eventhandling?");
+      infillProblem.Problem = Problem;
+
 
       //random
@@ -274,4 +250,6 @@
       table.Rows.Add(new DataRow(CurrentQualitiesRowResultName));
       Results.Add(new Result(QualitiesChartResultName, table));
+      if (BaselineVector != null && BaselineVector.Length == enc.Length)
+        Results.Add(new Result("BaselineValue", new DoubleValue(Evaluate(BaselineVector).Item2)));
 
       //initial samples
@@ -288,10 +266,11 @@
 
     protected override void Run(CancellationToken cancellationToken) {
-      for (ResultsIterations = 0; ResultsIterations < MaximumIterations; ResultsIterations++) {
+      for (ResultsIterations = 0; ResultsEvaluations < MaximumEvaluations; ResultsIterations++) {
         try {
           ResultsModel = BuildModel(cancellationToken);
+          if (ResultsModel == null) break;
           cancellationToken.ThrowIfCancellationRequested();
           for (var i = 0; i < GenerationSize; i++) {
-            var samplepoint = OptimizeInfillProblem();
+            var samplepoint = OptimizeInfillProblem(cancellationToken);
             var sample = Evaluate(samplepoint);
             Samples.Add(sample);
@@ -317,4 +296,6 @@
       InfillOptimizationAlgorithm.ProblemChanged += InfillOptimizationProblemChanged;
       InfillCriterionParameter.ValueChanged += InfillCriterionChanged;
+      InitialSamplesParameter.ToStringChanged += OnInitialSamplesChanged;
+
 
     }
@@ -324,4 +305,5 @@
       InfillOptimizationAlgorithm.ProblemChanged -= InfillOptimizationProblemChanged;
       InfillCriterionParameter.ValueChanged -= InfillCriterionChanged;
+      InitialSamplesParameter.ToStringChanged -= OnInitialSamplesChanged;
     }
     private void OnInfillOptimizationAlgorithmChanged(object sender, EventArgs args) {
@@ -343,4 +325,30 @@
       RegressionAlgorithm.Problem = new RegressionProblem();
     }
+    private void OnInitialSamplesChanged(object sender, EventArgs args) {
+      IRegressionProblemData samplesData = null;
+      using (var importTypeDialog = new RegressionImportTypeDialog()) {
+        if (importTypeDialog.ShowDialog() != DialogResult.OK) return;
+        samplesData = new RegressionCSVInstanceProvider().ImportData(importTypeDialog.Path, importTypeDialog.ImportType, importTypeDialog.CSVFormat);
+        InitialSamplesParameter.ToStringChanged -= OnInitialSamplesChanged;
+        InitialSamplesParameter.Value.Value = importTypeDialog.Path;
+        InitialSamplesParameter.ToStringChanged -= OnInitialSamplesChanged;
+
+      }
+
+
+
+      var solutions = new RealVector[samplesData.Dataset.Rows];
+      var qualities = new double[samplesData.Dataset.Rows];
+      var inputVariables = samplesData.InputVariables.CheckedItems.ToArray();
+      for (var i = 0; i < solutions.Length; i++) {
+        qualities[i] = samplesData.Dataset.GetDoubleValue(samplesData.TargetVariable, i);
+        solutions[i] = new RealVector(inputVariables.Length);
+        for (var j = 0; j < inputVariables.Length; j++) solutions[i][j] = samplesData.Dataset.GetDoubleValue(inputVariables[j].Value.Value, i);
+      }
+
+      SetInitialSamples(solutions, qualities);
+
+    }
+
     protected override void OnExecutionTimeChanged() {
       base.OnExecutionTimeChanged();
@@ -350,6 +358,6 @@
     }
     public override void Pause() {
-      if (InfillOptimizationAlgorithm.ExecutionState == ExecutionState.Started) InfillOptimizationAlgorithm.Pause();
-      if (RegressionAlgorithm.ExecutionState == ExecutionState.Started) RegressionAlgorithm.Pause();
+      if (InfillOptimizationAlgorithm.ExecutionState == ExecutionState.Started || InfillOptimizationAlgorithm.ExecutionState == ExecutionState.Paused) InfillOptimizationAlgorithm.Stop();
+      if (RegressionAlgorithm.ExecutionState == ExecutionState.Started || RegressionAlgorithm.ExecutionState == ExecutionState.Paused) RegressionAlgorithm.Stop();
       base.Pause();
     }
@@ -376,5 +384,5 @@
     }
     private IRegressionSolution BuildModel(CancellationToken cancellationToken) {
-      var dataset = EgoUtilities.GetDataSet(DataSamples.ToList());
+      var dataset = EgoUtilities.GetDataSet(DataSamples.ToList(), RemoveDuplicates);
       var problemdata = new RegressionProblemData(dataset, dataset.VariableNames.Where(x => !x.Equals("output")), "output");
       problemdata.TrainingPartition.Start = 0;
@@ -388,17 +396,48 @@
       var i = 0;
       IRegressionSolution solution = null;
-      double r2 = 0;
-      while ((solution == null || RegressionAlgorithm is GaussianProcessRegression && r2 < 0.95) && i++ < 100) {  //TODO: ask why GP degenerates to NaN so often
+
+      while (solution == null && i++ < 100) {  //TODO: Question: Why does GP degenerate to NaN so often? Answer: There is not even the slightest mitigation strategy for "almost duplicates" that ill-condition the covariance matrix.
         var results = EgoUtilities.SyncRunSubAlgorithm(RegressionAlgorithm, Random.Next(int.MaxValue));
         solution = results.Select(x => x.Value).OfType<IRegressionSolution>().SingleOrDefault();
-        r2 = solution?.TrainingRSquared ?? 0;
         cancellationToken.ThrowIfCancellationRequested();
       }
 
-      if (solution == null) throw new ArgumentException("The Algorithm did not return a Model");
+      //try creating a model with old hyperparameters and new dataset;
+      var gp = RegressionAlgorithm as GaussianProcessRegression;
+      var oldmodel = ResultsModel as GaussianProcessRegressionSolution;
+      if (gp != null && oldmodel != null) {
+        var n = Samples.First().Item1.Length;
+        var mean = (IMeanFunction)oldmodel.Model.MeanFunction.Clone();
+        var cov = (ICovarianceFunction)oldmodel.Model.CovarianceFunction.Clone();
+        if (mean.GetNumberOfParameters(n) != 0 || cov.GetNumberOfParameters(n) != 0) throw new ArgumentException("DEBUG: assumption about fixed paramters wrong");
+        var noise = 0.0;
+        double[] hyp = { noise };
+        try {
+          var model = new GaussianProcessModel(problemdata.Dataset, problemdata.TargetVariable,
+            problemdata.AllowedInputVariables, problemdata.TrainingIndices, hyp, mean, cov);
+          model.FixParameters();
+          var sol = new GaussianProcessRegressionSolution(model, problemdata);
+          if (solution == null || solution.TrainingMeanSquaredError > sol.TrainingMeanSquaredError) {
+            solution = sol;
+          }
+        }
+        catch (ArgumentException) { }
+      }
+
+
+      if (!ResultsQualities.Rows.ContainsKey("DEBUG: Degenerates")) ResultsQualities.Rows.Add(new DataRow("DEBUG: Degenerates"));
+      var row = ResultsQualities.Rows["DEBUG: Degenerates"];
+      row.Values.Add(i - 1);
+      if (solution == null) Results.Add(new Result("Status", new StringValue("The Algorithm did not return a Model")));
+      else {
+        if (!ResultsQualities.Rows.ContainsKey("DEBUG: RMSE")) ResultsQualities.Rows.Add(new DataRow("DEBUG: RMSE"));
+        row = ResultsQualities.Rows["DEBUG: RMSE"];
+        row.Values.Add(Math.Sqrt(solution.TrainingMeanSquaredError));
+      }
+
       RegressionAlgorithm.Runs.Clear();
       return solution;
     }
-    private RealVector OptimizeInfillProblem() {
+    private RealVector OptimizeInfillProblem(CancellationToken cancellationToken) {
       //parameterize and check InfillProblem
       var infillProblem = InfillOptimizationAlgorithm.Problem as InfillProblem;
@@ -406,6 +445,5 @@
       if (infillProblem.InfillCriterion != InfillCriterion) throw new ArgumentException("InfillCiriterion for Problem is not correct. Problem with Eventhandling?");
       if (infillProblem.Problem != Problem) throw new ArgumentException("Expensive real problem is not correctly set in InfillProblem. Problem with Eventhandling?");
-      infillProblem.RegressionSolution = ResultsModel;
-      if (MaximalDatasetSize > 0 && MaximalDatasetSize < Samples.Count) { infillProblem.Encoding.Bounds = EgoUtilities.GetBoundingBox(DataSamples.Select(x => x.Item1)); }
+      InfillCriterion.Initialize(ResultsModel, Problem.Maximization, infillProblem.Encoding);
 
       RealVector bestVector = null;
@@ -415,5 +453,5 @@
         //optimize
         var res = EgoUtilities.SyncRunSubAlgorithm(InfillOptimizationAlgorithm, Random.Next(int.MaxValue));
-
+        cancellationToken.ThrowIfCancellationRequested();
         //extract results
         if (!res.ContainsKey(BestInfillSolutionResultName)) throw new ArgumentException("The InfillOptimizationAlgorithm did not return a best solution");
Index: /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/EgoUtilities.cs
===================================================================
--- /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/EgoUtilities.cs	(revision 14817)
+++ /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/EgoUtilities.cs	(revision 14818)
@@ -94,24 +94,62 @@
 
     public static double GetEstimation(this IRegressionModel model, RealVector r) {
-      var dataset = GetDataSet(new[] { new Tuple<RealVector, double>(r, 0.0) });
+      var dataset = GetDataSet(new[] { new Tuple<RealVector, double>(r, 0.0) }, false);
       return model.GetEstimatedValues(dataset, new[] { 0 }).First();
     }
     public static double GetVariance(this IConfidenceRegressionModel model, RealVector r) {
-      var dataset = GetDataSet(new[] { new Tuple<RealVector, double>(r, 0.0) });
+      var dataset = GetDataSet(new[] { new Tuple<RealVector, double>(r, 0.0) }, false);
       return model.GetEstimatedVariances(dataset, new[] { 0 }).First();
     }
 
-    public static Dataset GetDataSet(IReadOnlyList<Tuple<RealVector, double>> samples) {
-      var n = samples[0].Item1.Length + 1;
-      var data = new double[samples.Count, n];
-      var names = new string[n - 1];
-      for (var i = 0; i < n; i++)
-        if (i < names.Length) {
-          names[i] = "input" + i;
-          for (var j = 0; j < samples.Count; j++) data[j, i] = samples[j].Item1[i];
-        } else
-          for (var j = 0; j < samples.Count; j++) data[j, n - 1] = samples[j].Item2;
+
+    public static double GetDoubleValue(this IDataset dataset, int i, int j) {
+      return dataset.GetDoubleValue("input" + j, i);
+    }
+    public static Dataset GetDataSet(IReadOnlyList<Tuple<RealVector, double>> samples, bool removeDuplicates) {
+      if (removeDuplicates)
+        samples = RemoveDuplicates(samples); //TODO duplicates require heteroskedasticity in Models
+
+
+      var dimensions = samples[0].Item1.Length + 1;
+      var data = new double[samples.Count, dimensions];
+      var names = new string[dimensions - 1];
+      for (var i = 0; i < names.Length; i++) names[i] = "input" + i;
+
+      for (var j = 0; j < samples.Count; j++) {
+        for (var i = 0; i < names.Length; i++) data[j, i] = samples[j].Item1[i];
+        data[j, dimensions - 1] = samples[j].Item2;
+
+      }
+
+
       return new Dataset(names.Concat(new[] { "output" }).ToArray(), data);
     }
+
+    private static IReadOnlyList<Tuple<RealVector, double>> RemoveDuplicates(IReadOnlyList<Tuple<RealVector, double>> samples) {
+      var res = new List<Tuple<RealVector, double, int>>();
+
+      foreach (var sample in samples) {
+        if (res.Count == 0) {
+          res.Add(new Tuple<RealVector, double, int>(sample.Item1, sample.Item2, 1));
+          continue;
+        }
+
+        var index = res.ArgMin(x => Euclidian(sample.Item1, x.Item1));
+        var d = Euclidian(res[index].Item1, sample.Item1);
+        if (d > 0.0001)
+          res.Add(new Tuple<RealVector, double, int>(sample.Item1, sample.Item2, 1));
+        else {
+          var t = res[index];
+          res.RemoveAt(index);
+          res.Add(new Tuple<RealVector, double, int>(t.Item1, t.Item2 + sample.Item2, t.Item3 + 1));
+        }
+      }
+      return res.Select(x => new Tuple<RealVector, double>(x.Item1, x.Item2 / x.Item3)).ToArray();
+    }
+
+    private static double Euclidian(IEnumerable<double> a, IEnumerable<double> b) {
+      return Math.Sqrt(a.Zip(b, (d, d1) => d - d1).Sum(d => d * d));
+    }
+
     public static DoubleMatrix GetBoundingBox(IEnumerable<RealVector> vectors) {
       DoubleMatrix res = null;
@@ -128,4 +166,6 @@
       return res;
     }
+
+
   }
 }
Index: /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/HeuristicLab.Algorithms.EGO-3.4.csproj
===================================================================
--- /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/HeuristicLab.Algorithms.EGO-3.4.csproj	(revision 14817)
+++ /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/HeuristicLab.Algorithms.EGO-3.4.csproj	(revision 14818)
@@ -92,4 +92,12 @@
       <HintPath>..\..\..\trunk\sources\bin\HeuristicLab.Problems.Instances-3.3.dll</HintPath>
     </Reference>
+    <Reference Include="HeuristicLab.Problems.Instances.DataAnalysis-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\..\trunk\sources\bin\HeuristicLab.Problems.Instances.DataAnalysis-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Problems.Instances.DataAnalysis.Views-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\..\trunk\sources\bin\HeuristicLab.Problems.Instances.DataAnalysis.Views-3.3.dll</HintPath>
+    </Reference>
     <Reference Include="HeuristicLab.Random-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
@@ -98,4 +106,5 @@
     <Reference Include="System" />
     <Reference Include="System.Core" />
+    <Reference Include="System.Windows.Forms" />
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />
@@ -108,6 +117,9 @@
     <Compile Include="EgoUtilities.cs" />
     <Compile Include="EfficientGlobalOptimizationAlgorithm.cs" />
+    <Compile Include="InfillCriteria\AugmentedExpectedImprovement.cs" />
+    <Compile Include="InfillCriteria\ExpectedQuantileImprovement.cs" />
+    <Compile Include="InfillCriteria\PluginExpectedImprovement.cs" />
     <Compile Include="InfillCriteria\RobustImprovement.cs" />
-    <Compile Include="InfillCriteria\ConfidenceBound.cs" />
+    <Compile Include="InfillCriteria\MinimalQuantileCriterium.cs" />
     <Compile Include="InfillCriteria\ExpectedQuality.cs" />
     <Compile Include="InfillCriteria\InfillCriterionBase.cs" />
Index: /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/AugmentedExpectedImprovement.cs
===================================================================
--- /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/AugmentedExpectedImprovement.cs	(revision 14818)
+++ /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/AugmentedExpectedImprovement.cs	(revision 14818)
@@ -0,0 +1,98 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+// ReSharper disable once CheckNamespace
+namespace HeuristicLab.Algorithms.EGO {
+
+  [StorableClass]
+  [Item("AugmentedExpectedImprovement", "Noisy InfillCriterion, Extension of the Expected Improvement as described in\n Global optimization of stochastic black-box systems via sequential kriging meta-models.\r\nHuang, D., Allen, T., Notz, W., Zeng, N.")]
+  public class AugmentedExpectedImprovement : ExpectedImprovement {
+
+
+
+    #region Parameternames
+
+    public const string AlphaParameterName = "Alpha";
+
+    #endregion
+
+    #region Parameters
+
+    public IValueParameter<DoubleValue> AlphaParameter => Parameters[AlphaParameterName] as IValueParameter<DoubleValue>;
+
+    #endregion
+
+    #region Properties
+
+    public double Alpha => AlphaParameter.Value.Value;
+    [Storable]
+    private double Tau;
+
+    #endregion
+
+
+    #region HL-Constructors, Serialization and Cloning 
+    [StorableConstructor]
+    private AugmentedExpectedImprovement(bool deserializing) : base(deserializing) { }
+
+    private AugmentedExpectedImprovement(AugmentedExpectedImprovement original, Cloner cloner) : base(original, cloner) {
+      Tau = original.Tau;
+    }
+
+    public AugmentedExpectedImprovement() {
+      Parameters.Add(new ValueParameter<DoubleValue>(AlphaParameterName, "The Alpha value specifiying the robustness of the \"effective best solution\". Recommended value is 1", new DoubleValue(1.0)));
+
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new AugmentedExpectedImprovement(this, cloner);
+    }
+    #endregion
+
+    public override double Evaluate(RealVector vector) {
+      var model = RegressionSolution.Model as IConfidenceRegressionModel;
+      return base.Evaluate(vector) * (1 - Tau / Math.Sqrt(model.GetVariance(vector) + Tau * Tau));
+    }
+
+    protected override void Initialize() {
+      if (ExpensiveMaximization) throw new NotImplementedException("AugmentedExpectedImprovement for maximization not yet implemented");
+      var solution = RegressionSolution as IConfidenceRegressionSolution;
+      if (solution == null) throw new ArgumentException("can not calculate Augmented EI without a regression solution providing confidence values");
+
+      Tau = RegressionSolution.EstimatedTrainingValues.Zip(RegressionSolution.ProblemData.TargetVariableTrainingValues, (d, d1) => Math.Abs(d - d1)).Average();
+      var xss = new RealVector(Encoding.Length);
+      var xssIndex = solution.EstimatedTrainingValues.Zip(solution.EstimatedTrainingValues, (m, s2) => m + Alpha * Math.Sqrt(s2)).ArgMin(x => x);
+      var i = solution.ProblemData.TrainingIndices.ToArray()[xssIndex];
+      for (var j = 0; j < Encoding.Length; j++) xss[j] = solution.ProblemData.Dataset.GetDoubleValue(i, j);
+
+      YMin = RegressionSolution.Model.GetEstimation(xss);
+    }
+  }
+}
Index: /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/ExpectedImprovement.cs
===================================================================
--- /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/ExpectedImprovement.cs	(revision 14817)
+++ /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/ExpectedImprovement.cs	(revision 14818)
@@ -42,25 +42,23 @@
 
     #region ParameterProperties
-    public IFixedValueParameter<DoubleValue> ExploitationWeightParameter
-    {
-      get { return Parameters[ExploitationWeightParameterName] as IFixedValueParameter<DoubleValue>; }
-    }
+    public IFixedValueParameter<DoubleValue> ExploitationWeightParameter => Parameters[ExploitationWeightParameterName] as IFixedValueParameter<DoubleValue>;
+
     #endregion
 
     #region Properties
-    private double ExploitationWeight
-    {
-      get { return ExploitationWeightParameter.Value.Value; }
-    }
+    protected double ExploitationWeight => ExploitationWeightParameter.Value.Value;
+
+    [Storable]
+    protected double YMin;
     #endregion
 
     #region HL-Constructors, Serialization and Cloning 
     [StorableConstructor]
-    private ExpectedImprovement(bool deserializing) : base(deserializing) { }
+    protected ExpectedImprovement(bool deserializing) : base(deserializing) { }
     [StorableHook(HookType.AfterDeserialization)]
     private void AfterDeserialization() {
       RegisterEventhandlers();
     }
-    private ExpectedImprovement(ExpectedImprovement original, Cloner cloner) : base(original, cloner) {
+    protected ExpectedImprovement(ExpectedImprovement original, Cloner cloner) : base(original, cloner) {
       RegisterEventhandlers();
     }
@@ -74,16 +72,20 @@
     #endregion
 
-    public override double Evaluate(IRegressionSolution solution, RealVector vector, bool maximization) {
-      if (maximization) throw new NotImplementedException("Expected Improvement for maximization not yet implemented");
-      var model = solution.Model as IConfidenceRegressionModel;
-      if (model == null) throw new ArgumentException("can not calculate EI without confidence measure");
+    public override double Evaluate(RealVector vector) {
+      var model = RegressionSolution.Model as IConfidenceRegressionModel;
       var yhat = model.GetEstimation(vector);
-      var min = solution.ProblemData.TargetVariableTrainingValues.Min();
       var s = Math.Sqrt(model.GetVariance(vector));
-      return GetEstimatedImprovement(min, yhat, s, ExploitationWeight);
+      return GetEstimatedImprovement(YMin, yhat, s, ExploitationWeight);
     }
 
-    public override bool Maximization(bool expensiveProblemMaximization) {
+    public override bool Maximization() {
       return true;
+    }
+
+    protected override void Initialize() {
+      if (ExpensiveMaximization) throw new NotImplementedException("Expected Improvement for maximization not yet implemented");
+      var model = RegressionSolution.Model as IConfidenceRegressionModel;
+      if (model == null) throw new ArgumentException("can not calculate EI without confidence measure");
+      YMin = RegressionSolution.ProblemData.TargetVariableTrainingValues.Min();
     }
 
@@ -104,5 +106,5 @@
 
     #region Helpers
-    private static double GetEstimatedImprovement(double ymin, double yhat, double s, double w) {
+    protected static double GetEstimatedImprovement(double ymin, double yhat, double s, double w) {
       if (Math.Abs(s) < double.Epsilon) return 0;
       var val = (ymin - yhat) / s;
Index: /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/ExpectedQuality.cs
===================================================================
--- /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/ExpectedQuality.cs	(revision 14817)
+++ /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/ExpectedQuality.cs	(revision 14818)
@@ -24,5 +24,4 @@
 using HeuristicLab.Encodings.RealVectorEncoding;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using HeuristicLab.Problems.DataAnalysis;
 
 // ReSharper disable once CheckNamespace
@@ -42,6 +41,13 @@
     #endregion
 
-    public override double Evaluate(IRegressionSolution solution, RealVector vector, bool maximization) {
-      return solution.Model.GetEstimation(vector);
+    public override double Evaluate(RealVector vector) {
+      return RegressionSolution.Model.GetEstimation(vector);
+    }
+
+    public override bool Maximization() {
+      return ExpensiveMaximization;
+    }
+
+    protected override void Initialize() {
     }
   }
Index: /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/ExpectedQuantileImprovement.cs
===================================================================
--- /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/ExpectedQuantileImprovement.cs	(revision 14818)
+++ /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/ExpectedQuantileImprovement.cs	(revision 14818)
@@ -0,0 +1,103 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+// ReSharper disable once CheckNamespace
+namespace HeuristicLab.Algorithms.EGO {
+
+  [StorableClass]
+  [Item("ExpectedQuantileImprovement", "Noisy InfillCriterion, Extension of the Expected Improvement as described in \n Noisy expectedimprovement and on - line computation time allocation for the optimization of simulators with tunable fidelitys\r\nPicheny, V., Ginsbourger, D., Richet, Y")]
+  public class ExpectedQuantileImprovement : ExpectedImprovement {
+
+    #region Parameternames
+    public const string AlphaParameterName = "Alpha";
+    public const string MaxEvaluationsParameterName = "MaxEvaluations";
+    #endregion
+
+    #region Parameters
+    public IFixedValueParameter<DoubleValue> AlphaParameter => Parameters[AlphaParameterName] as IFixedValueParameter<DoubleValue>;
+    public IValueParameter<IntValue> MaxEvaluationsParameter => Parameters[MaxEvaluationsParameterName] as IValueParameter<IntValue>;
+    #endregion
+
+    #region Properties
+
+    public int MaxEvaluations => MaxEvaluationsParameter.Value.Value;
+    public double Alpha => AlphaParameter.Value.Value;
+    [Storable]
+    private double Tau;
+
+    #endregion
+
+    #region HL-Constructors, Serialization and Cloning 
+    [StorableConstructor]
+    private ExpectedQuantileImprovement(bool deserializing) : base(deserializing) { }
+
+    private ExpectedQuantileImprovement(ExpectedQuantileImprovement original, Cloner cloner) : base(original, cloner) {
+      Tau = original.Tau;
+    }
+
+    public ExpectedQuantileImprovement() {
+      Parameters.Add(new FixedValueParameter<DoubleValue>(AlphaParameterName, "The Alpha value specifiying the robustness of the \"effective best solution\". Recommended value is 1.0", new DoubleValue(1.0)));
+      Parameters.Add(new ValueParameter<IntValue>(MaxEvaluationsParameterName, "The maximum number of evaluations allowed for EGO", new IntValue(100)));
+      MaxEvaluationsParameter.Hidden = true;
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new ExpectedQuantileImprovement(this, cloner);
+    }
+    #endregion
+
+    public override double Evaluate(RealVector vector) {
+      var model = RegressionSolution.Model as IConfidenceRegressionModel;
+      var s2 = model.GetVariance(vector);
+
+      var yhat = model.GetEstimation(vector) + Alpha * Math.Sqrt(Tau * s2 / (Tau + s2));
+      var s = Math.Sqrt(s2 * s2 / (Tau + s2));
+
+      return GetEstimatedImprovement(YMin, yhat, s, ExploitationWeight);
+    }
+
+    protected override void Initialize() {
+      if (ExpensiveMaximization) throw new NotImplementedException("AugmentedExpectedImprovement for maximization not yet implemented");
+      var solution = RegressionSolution as IConfidenceRegressionSolution;
+      if (solution == null) throw new ArgumentException("can not calculate Augmented EI without a regression solution providing confidence values");
+
+      Tau = RegressionSolution.EstimatedTrainingValues.Zip(RegressionSolution.ProblemData.TargetVariableTrainingValues, (d, d1) => Math.Abs(d - d1)).Average();
+      Tau = Tau * Tau / (MaxEvaluations - RegressionSolution.ProblemData.Dataset.Rows + 1);
+
+      var xss = new RealVector(Encoding.Length);
+      var xssIndex = solution.EstimatedTrainingVariances.Zip(solution.EstimatedTrainingVariances, (m, s2) => m + Alpha * Math.Sqrt(s2)).ArgMin(x => x);
+      var i = solution.ProblemData.TrainingIndices.ToArray()[xssIndex];
+      for (var j = 0; j < Encoding.Length; j++) xss[j] = solution.ProblemData.Dataset.GetDoubleValue(i, j);
+
+      YMin = RegressionSolution.Model.GetEstimation(xss);
+    }
+
+  }
+}
Index: /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/InfillCriterionBase.cs
===================================================================
--- /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/InfillCriterionBase.cs	(revision 14817)
+++ /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/InfillCriterionBase.cs	(revision 14818)
@@ -23,19 +23,40 @@
 using HeuristicLab.Core;
 using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
 
 // ReSharper disable once CheckNamespace
 namespace HeuristicLab.Algorithms.EGO {
-
+  [StorableClass]
   public abstract class InfillCriterionBase : ParameterizedNamedItem, IInfillCriterion {
 
+    [Storable]
+    protected IRegressionSolution RegressionSolution;
+    [Storable]
+    protected bool ExpensiveMaximization;
+    [Storable]
+    protected RealVectorEncoding Encoding;
+
     protected InfillCriterionBase(bool deserializing) : base(deserializing) { }
-    protected InfillCriterionBase(InfillCriterionBase original, Cloner cloner) : base(original, cloner) { }
+
+    protected InfillCriterionBase(InfillCriterionBase original, Cloner cloner) : base(original, cloner) {
+      RegressionSolution = cloner.Clone(original.RegressionSolution);
+      ExpensiveMaximization = original.ExpensiveMaximization;
+      Encoding = cloner.Clone(original.Encoding);
+    }
     protected InfillCriterionBase() { }
 
-    public abstract double Evaluate(IRegressionSolution model, RealVector vector, bool maximization);
-    public virtual bool Maximization(bool expensiveProblemMaximization) {
-      return expensiveProblemMaximization;
+    public abstract double Evaluate(RealVector vector);
+    public abstract bool Maximization();
+
+    public void Initialize(IRegressionSolution solution, bool expensiveMaximization, RealVectorEncoding encoding) {
+      RegressionSolution = solution;
+      ExpensiveMaximization = expensiveMaximization;
+      Encoding = encoding;
+      Initialize();
     }
+
+    protected abstract void Initialize();
+
   }
 }
Index: /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/MinimalQuantileCriterium.cs
===================================================================
--- /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/MinimalQuantileCriterium.cs	(revision 14818)
+++ /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/MinimalQuantileCriterium.cs	(revision 14818)
@@ -0,0 +1,80 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+// ReSharper disable once CheckNamespace
+namespace HeuristicLab.Algorithms.EGO {
+
+  [StorableClass]
+  [Item("MinimalQuantileCriterium", "Adding or Subtracting the variance * factor to the model estimation")]
+  public class MinimalQuantileCriterium : InfillCriterionBase {
+
+    #region ParameterNames
+    private const string ConfidenceWeightParameterName = "ConfidenceWeight";
+    #endregion
+
+    #region ParameterProperties
+    public IFixedValueParameter<DoubleValue> ConfidenceWeightParameter => Parameters[ConfidenceWeightParameterName] as IFixedValueParameter<DoubleValue>;
+
+    #endregion
+
+    #region Properties
+    private double ConfidenceWeight => ConfidenceWeightParameter.Value.Value;
+
+    #endregion
+
+    #region HL-Constructors, Serialization and Cloning 
+    [StorableConstructor]
+    private MinimalQuantileCriterium(bool deserializing) : base(deserializing) { }
+    private MinimalQuantileCriterium(MinimalQuantileCriterium original, Cloner cloner) : base(original, cloner) { }
+    public MinimalQuantileCriterium() {
+      Parameters.Add(new FixedValueParameter<DoubleValue>(ConfidenceWeightParameterName, "A value between 0 and 1 indicating the focus on exploration (0) or exploitation (1)", new DoubleValue(0.5)));
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MinimalQuantileCriterium(this, cloner);
+    }
+    #endregion
+
+    public override double Evaluate(RealVector vector) {
+      var model = RegressionSolution.Model as IConfidenceRegressionModel;
+      var yhat = model.GetEstimation(vector);
+      var s = Math.Sqrt(model.GetVariance(vector)) * ConfidenceWeight;
+      return ExpensiveMaximization ? yhat + s : yhat - s;
+    }
+
+    public override bool Maximization() {
+      return ExpensiveMaximization;
+    }
+
+    protected override void Initialize() {
+      var model = RegressionSolution.Model as IConfidenceRegressionModel;
+      if (model == null) throw new ArgumentException("can not calculate EI without confidence measure");
+    }
+  }
+}
Index: /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/PluginExpectedImprovement.cs
===================================================================
--- /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/PluginExpectedImprovement.cs	(revision 14818)
+++ /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/PluginExpectedImprovement.cs	(revision 14818)
@@ -0,0 +1,54 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+// ReSharper disable once CheckNamespace
+namespace HeuristicLab.Algorithms.EGO {
+
+  [StorableClass]
+  [Item("PluginExpectedImprovement", "Noisy InfillCriterion, Extension of the Expected Improvement by using the minimal prediction on the observed points\n rather than the minimal observed value as described in \n Global optimization based on noisy evaluations: An empirical study of two statistical approaches\r\nEmmanuel Vazqueza, Julien Villemonteixb, Maryan Sidorkiewiczb and Éric Walterc")]
+  public class PluginExpectedImprovement : ExpectedImprovement {
+
+
+    #region HL-Constructors, Serialization and Cloning 
+    [StorableConstructor]
+    private PluginExpectedImprovement(bool deserializing) : base(deserializing) { }
+    private PluginExpectedImprovement(PluginExpectedImprovement original, Cloner cloner) : base(original, cloner) { }
+    public PluginExpectedImprovement() { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new PluginExpectedImprovement(this, cloner);
+    }
+    #endregion
+
+    protected override void Initialize() {
+      if (ExpensiveMaximization) throw new NotImplementedException("PluginExpectedImprovement for maximization not yet implemented");
+      var model = RegressionSolution.Model as IConfidenceRegressionModel;
+      if (model == null) throw new ArgumentException("can not calculate EI without confidence measure");
+      YMin = RegressionSolution.EstimatedTrainingValues.Min();
+    }
+  }
+}
Index: /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/RobustImprovement.cs
===================================================================
--- /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/RobustImprovement.cs	(revision 14817)
+++ /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/InfillCriteria/RobustImprovement.cs	(revision 14818)
@@ -21,4 +21,6 @@
 
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -33,45 +35,125 @@
 
   [StorableClass]
-  [Item("ConfidenceBoundMeassure", "Adding or Subtracting the variance * factor to the model estimation")]
-  public class ConfidenceBound : InfillCriterionBase {
+  [Item("RobustImprovementMeassure", "Adding or Subtracting the variance * factor to the model estimation")]
+  public class RobustImprovement : InfillCriterionBase {
 
     #region ParameterNames
-    private const string ConfidenceWeightParameterName = "ConfidenceWeight";
+    private const string KParameterName = "NearestNeighbours";
     #endregion
 
     #region ParameterProperties
-    public IFixedValueParameter<DoubleValue> ConfidenceWeightParameter
-    {
-      get { return Parameters[ConfidenceWeightParameterName] as IFixedValueParameter<DoubleValue>; }
-    }
+    public IFixedValueParameter<IntValue> KParameter => Parameters[KParameterName] as IFixedValueParameter<IntValue>;
+
     #endregion
 
     #region Properties
-    private double ConfidenceWeight
-    {
-      get { return ConfidenceWeightParameter.Value.Value; }
-    }
+    private int K => KParameter.Value.Value;
+
+    [Storable]
+    private double MaxSolutionDist;
+
+    [Storable]
+    //TODO use VP-Tree instead of array
+    private RealVector[] Data;
     #endregion
 
     #region HL-Constructors, Serialization and Cloning 
     [StorableConstructor]
-    private ConfidenceBound(bool deserializing) : base(deserializing) { }
-    private ConfidenceBound(ConfidenceBound original, Cloner cloner) : base(original, cloner) { }
-    public ConfidenceBound() {
-      Parameters.Add(new FixedValueParameter<DoubleValue>(ConfidenceWeightParameterName, "A value between 0 and 1 indicating the focus on exploration (0) or exploitation (1)", new DoubleValue(0.5)));
+    private RobustImprovement(bool deserializing) : base(deserializing) { }
+
+    private RobustImprovement(RobustImprovement original, Cloner cloner) : base(original, cloner) {
+      MaxSolutionDist = original.MaxSolutionDist;
+      Data = original.Data != null ? original.Data.Select(cloner.Clone).ToArray() : null;
+    }
+    public RobustImprovement() {
+      Parameters.Add(new FixedValueParameter<IntValue>(KParameterName, "A value larger than 0 indicating how many nearestNeighbours shall be used to determine the RI meassure", new IntValue(3)));
     }
     public override IDeepCloneable Clone(Cloner cloner) {
-      return new ConfidenceBound(this, cloner);
+      return new RobustImprovement(this, cloner);
     }
     #endregion
 
-    public override double Evaluate(IRegressionSolution solution, RealVector vector, bool maximization) {
-      var model = solution.Model as IConfidenceRegressionModel;
+
+    public override double Evaluate(RealVector vector) {
+      List<RealVector> nearestNeighbours;
+      List<double> distances;
+      Search(vector, K, out nearestNeighbours, out distances);
+      var distVectors = nearestNeighbours.Select(x => Minus(x, vector)).ToList();
+      var sum = 0.0;
+      var wsum = 1.0; //weights for angular distance
+      var used = new HashSet<RealVector>();
+      foreach (var distVector in distVectors) {
+        var d = Math.Pow(distances[used.Count], 0.5);
+        if (used.Count == 0) {
+          sum += d;
+        } else {
+          var w = used.Select(x => Angular(distVector, x)).Min();
+          sum += w * d;
+          wsum += w;
+        }
+        used.Add(distVector);
+      }
+      sum /= wsum * MaxSolutionDist; //normalize
+      return sum;
+    }
+    public override bool Maximization() {
+      return ExpensiveMaximization;
+    }
+    protected override void Initialize() {
+      var model = RegressionSolution.Model as IConfidenceRegressionModel;
       if (model == null) throw new ArgumentException("can not calculate EI without confidence measure");
-      var yhat = model.GetEstimation(vector);
-      var s = Math.Sqrt(model.GetVariance(vector)) * ConfidenceWeight;
-      return maximization ? yhat + s : yhat - s;
+      Data = new RealVector[RegressionSolution.ProblemData.Dataset.Rows];
+      for (var i = 0; i < Data.Length; i++) {
+        Data[i] = new RealVector(Encoding.Length);
+        for (var j = 0; j < Encoding.Length; j++)
+          Data[i][j] = RegressionSolution.ProblemData.Dataset.GetDoubleValue(i, j);
+      }
+
+      var maxSolution = new double[Encoding.Length];
+      var minSolution = new double[Encoding.Length];
+      for (var i = 0; i < Encoding.Length; i++) {
+        var j = i % Encoding.Bounds.Rows;
+        maxSolution[i] = Encoding.Bounds[j, 1];
+        minSolution[i] = Encoding.Bounds[j, 0];
+      }
+      MaxSolutionDist = Euclidian(maxSolution, minSolution) / Data.Length;
     }
 
+    #region Helpers
+    private static double Euclidian(IEnumerable<double> a, IEnumerable<double> b) {
+      return Math.Sqrt(a.Zip(b, (d, d1) => d - d1).Sum(d => d * d));
+    }
+    private static double Angular(RealVector a, RealVector b) {
+      var innerProduct = a.Zip(b, (x, y) => x * y).Sum();
+      var res = Math.Acos(innerProduct / (Norm(a) * Norm(b))) / Math.PI;
+      return double.IsNaN(res) ? 0 : res;
+    }
+    private static double Norm(IEnumerable<double> a) {
+      return Math.Sqrt(a.Sum(d => d * d));
+    }
+    private static RealVector Minus(RealVector a, RealVector b) {
+      return new RealVector(a.Zip(b, (d, d1) => d - d1).ToArray());
+    }
+
+    private void Search(RealVector vector, int k, out List<RealVector> nearestNeighbours, out List<double> distances) {
+      var neighbours = new SortedList<double, RealVector>(new DuplicateKeyComparer<double>());
+      foreach (var n in Data) neighbours.Add(Euclidian(n, vector), n);
+      nearestNeighbours = new List<RealVector>();
+
+      distances = new List<double>();
+      foreach (var entry in neighbours) {
+        nearestNeighbours.Add(entry.Value);
+        distances.Add(entry.Key);
+        if (distances.Count == k) break;
+      }
+    }
+    #endregion
+
+    public class DuplicateKeyComparer<TKey> : IComparer<TKey> where TKey : IComparable {
+      public int Compare(TKey x, TKey y) {
+        var result = x.CompareTo(y);
+        return result == 0 ? 1 : result;
+      }
+    }
   }
 }
Index: /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/Interfaces/IInfillCriterion.cs
===================================================================
--- /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/Interfaces/IInfillCriterion.cs	(revision 14817)
+++ /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/Interfaces/IInfillCriterion.cs	(revision 14818)
@@ -27,6 +27,7 @@
 namespace HeuristicLab.Algorithms.EGO {
   public interface IInfillCriterion : INamedItem {
-    double Evaluate(IRegressionSolution model, RealVector vector, bool maximization);
-    bool Maximization(bool expensiveProblemMaximization);
+    double Evaluate(RealVector vector);
+    bool Maximization();
+    void Initialize(IRegressionSolution solution, bool expensiveMaximization, RealVectorEncoding encoding);
   }
 }
Index: /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/Interfaces/ISurrogateAlgorithm.cs
===================================================================
--- /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/Interfaces/ISurrogateAlgorithm.cs	(revision 14817)
+++ /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/Interfaces/ISurrogateAlgorithm.cs	(revision 14818)
@@ -26,5 +26,5 @@
 namespace HeuristicLab.Algorithms.EGO {
   public interface ISurrogateAlgorithm<T> : IAlgorithm where T : IDeepCloneable {
-    void SetInitialSamples(T[] i, double[] qualities);
+    void SetInitialSamples(T[] solutions, double[] qualities);
   }
 }
Index: /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/Problems/InfillProblem.cs
===================================================================
--- /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/Problems/InfillProblem.cs	(revision 14817)
+++ /branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/Problems/InfillProblem.cs	(revision 14818)
@@ -27,5 +27,4 @@
 using HeuristicLab.Optimization;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using HeuristicLab.Problems.DataAnalysis;
 
 namespace HeuristicLab.Algorithms.EGO {
@@ -34,5 +33,5 @@
   public sealed class InfillProblem : SingleObjectiveBasicProblem<RealVectorEncoding> {
 
-    public override bool Maximization => true;
+    public override bool Maximization => true;  //This is necessary because algorithms do not expect the maximization to change
 
     #region Properties;
@@ -41,7 +40,4 @@
     [Storable]
     private SingleObjectiveBasicProblem<IEncoding> problem;
-    [Storable]
-    private IRegressionSolution regressionSolution;
-
 
     public IInfillCriterion InfillCriterion
@@ -57,13 +53,10 @@
         problem = value;
         if (problem == null) return;
-        Encoding = problem.Encoding as RealVectorEncoding;
+        var enc = problem.Encoding as RealVectorEncoding;
+        if (enc == null) throw new ArgumentException("EGO can not be performed on non-RealVectorEncodings");
+        Encoding = enc;
         SolutionCreator = new UniformRandomRealVectorCreator();//ignore Problem specific Solution Creation
-        if (Encoding == null) throw new ArgumentException("EGO can not be performed on non-RealVectorEncodings");
+
       }
-    }
-    public IRegressionSolution RegressionSolution
-    {
-      get { return regressionSolution; }
-      set { regressionSolution = value; }
     }
     #endregion
@@ -75,5 +68,4 @@
       infillCriterion = cloner.Clone(original.InfillCriterion);
       problem = cloner.Clone(original.Problem);
-      regressionSolution = cloner.Clone(original.regressionSolution);
     }
     public InfillProblem() { }
@@ -82,12 +74,11 @@
 
     public override double Evaluate(Individual individual, IRandom r) {
-      var q = InfillCriterion.Evaluate(RegressionSolution, individual.RealVector(), Problem.Maximization);
-      return InfillCriterion.Maximization(Problem.Maximization) ? q : -q; //This is necessary because Maximization is not supposed to change on a normal problem
+      var q = InfillCriterion.Evaluate(individual.RealVector());
+      return InfillCriterion.Maximization() ? q : -q;
     }
     public override void Analyze(Individual[] individuals, double[] qualities, ResultCollection results, IRandom random) {
       base.Analyze(individuals, qualities, results, random);
       var best = qualities.ArgMax(x => x);
-      var qnew = InfillCriterion.Maximization(Problem.Maximization) ? qualities[best] : -qualities[best];
-      //var best = Maximization ? qualities.ArgMax(x => x) : qualities.ArgMin(x => x);
+      var qnew = InfillCriterion.Maximization() ? qualities[best] : -qualities[best];
       const string qname = EfficientGlobalOptimizationAlgorithm.BestInfillQualityResultName;
       const string sname = EfficientGlobalOptimizationAlgorithm.BestInfillSolutionResultName;
@@ -99,8 +90,7 @@
       var qold = results[qname].Value as DoubleValue;
       if (qold == null) throw new ArgumentException("Old best quality is not a double value. Conflicting Analyzers?");
-      if (qold.Value >= qnew == InfillCriterion.Maximization(Problem.Maximization)) return;
+      if (qold.Value >= qnew == InfillCriterion.Maximization()) return;
       results[sname].Value = (RealVector)individuals[best].RealVector().Clone();
       qold.Value = qnew;
-
     }
   }
