source: branches/EfficientGlobalOptimization/HeuristicLab.Algorithms.EGO/AdaptiveSamplingAlgorithm.cs @ 15258

using System;
using System.Collections.Generic;
using System.Linq;
using HeuristicLab.Algorithms.DataAnalysis;
using HeuristicLab.Analysis;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Encodings.RealVectorEncoding;
using HeuristicLab.Operators;
using HeuristicLab.Optimization;
using HeuristicLab.Optimization.Operators;
using HeuristicLab.Parameters;
using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
using HeuristicLab.Problems.DataAnalysis;
using HeuristicLab.Random;

namespace HeuristicLab.Algorithms.EGO {
  [Item("Adaptive Sampling Algorithm (EGO)", "An adaptive sampling algorithm.")]
  [Creatable(CreatableAttribute.Categories.SingleSolutionAlgorithms, Priority = 150)]
  [StorableClass]
  public class AdaptiveSamplingAlgorithm : HeuristicOptimizationEngineAlgorithm, IStorableContent {
    public string Filename { get; set; }

    #region Problem Properties
    public override Type ProblemType => typeof(IHeuristicOptimizationProblem);
    private ISingleObjectiveHeuristicOptimizationProblem SingleObjectiveProblem => Problem as ISingleObjectiveHeuristicOptimizationProblem;
    #endregion

    #region Parameter Names
    private const string InfillCriterionParameterName = "InfillCriterion";
    private const string InitialEvaluationsParameterName = "Initial Evaluations";
    private const string TerminatorParameterName = "Terminator";
    private const string SeedParameterName = "Seed";
    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
    private const string RegressionAlgorithmParameterName = "RegressionAlgorithm";
    private const string InfillOptimizationAlgorithmParameterName = "InfillOptimizationAlgorithm";
    private const string AnalyzerParameterName = "Analyzers";
    private const string RemoveDuplicatesParamterName = "PerturbDuplicates";
    private const string MaxModelSizeParameterName = "Maximal Model Size";
    #endregion

    #region Parameter Properties
    public IFixedValueParameter<IntValue> SeedParameter => (IFixedValueParameter<IntValue>)Parameters[SeedParameterName];
    public IFixedValueParameter<BoolValue> SetSeedRandomlyParameter => (IFixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName];
    public IFixedValueParameter<MultiTerminator> TerminatorParameter => (IFixedValueParameter<MultiTerminator>)Parameters[TerminatorParameterName];
    public IFixedValueParameter<IntValue> InitialEvaluationsParameter => (IFixedValueParameter<IntValue>)Parameters[InitialEvaluationsParameterName];
    public IConstrainedValueParameter<IInfillCriterion> InfillCriterionParameter => Parameters[InfillCriterionParameterName] as IConstrainedValueParameter<IInfillCriterion>;
    public IValueParameter<Algorithm> InfillOptimizationAlgorithmParameter => Parameters[InfillOptimizationAlgorithmParameterName] as IValueParameter<Algorithm>;
    public IValueParameter<IDataAnalysisAlgorithm<IRegressionProblem>> RegressionAlgorithmParameter => Parameters[RegressionAlgorithmParameterName] as IValueParameter<IDataAnalysisAlgorithm<IRegressionProblem>>;
    private IFixedValueParameter<MultiAnalyzer> AnalyzerParameter => (IFixedValueParameter<MultiAnalyzer>)Parameters[AnalyzerParameterName];
    public IFixedValueParameter<BoolValue> RemoveDuplicatesParameter => (IFixedValueParameter<BoolValue>)Parameters[RemoveDuplicatesParamterName];
    public IFixedValueParameter<IntValue> MaxModelSizeParameter => (IFixedValueParameter<IntValue>)Parameters[MaxModelSizeParameterName];
    #endregion

    #region Properties
    public int Seed => SeedParameter.Value.Value;
    public bool SetSeedRandomly => SetSeedRandomlyParameter.Value.Value;
    public MultiTerminator Terminator => TerminatorParameter.Value;
    public MultiAnalyzer Analyzer => AnalyzerParameter.Value;
    public int InitialEvaluations => InitialEvaluationsParameter.Value.Value;
    public IInfillCriterion InfillCriterion => InfillCriterionParameter.Value;
    public Algorithm InfillOptimizationAlgorithm => InfillOptimizationAlgorithmParameter.Value;
    public IDataAnalysisAlgorithm<IRegressionProblem> RegressionAlgorithm => RegressionAlgorithmParameter.Value;
    public bool RemoveDuplicates => RemoveDuplicatesParameter.Value.Value;
    public int MaxModelSize => MaxModelSizeParameter.Value.Value;
    #endregion

    #region Helper Properties
    private SolutionsCreator SolutionsCreator => OperatorGraph.Iterate().OfType<SolutionsCreator>().First();
    #endregion

    #region Preconfigured Analyzers
    [Storable]
    private BestAverageWorstQualityAnalyzer singleObjectiveQualityAnalyzer;
    [Storable]
    private CorrelationAnalyzer correlationAnalyzer;
    [Storable]
    private EvaluatedSolutionsAnalyzer sampleAnalyzer;
    [Storable]
    private ModelQualityAnalyzer modelAnalyzer;
    [Storable]
    private VariableVariabilityAnalyzer variabilityAnalyzer;
    [Storable]
    private FitnessClusteringAnalyzer fitnessClusterAnalyzer;
    #endregion

    #region Preconfigured Terminators
    [Storable]
    private ComparisonTerminator<IntValue> evaluationsTerminator;
    [Storable]
    private ExecutionTimeTerminator executionTimeTerminator;
    [Storable]
    private SingleObjectiveQualityTerminator qualityTerminator;
    #endregion

    #region Problem-dependent Operators
    [Storable]
    private SampleCollector sampleCollector;
    [Storable]
    private InfillSolver infillSolver;
    [Storable]
    private SolutionsCreator infillSolutionsCreator;
    #endregion

    #region Constructors
    [StorableConstructor]
    protected AdaptiveSamplingAlgorithm(bool deserializing) : base(deserializing) { }
    [StorableHook(HookType.AfterDeserialization)]
    private void AfterDeserialization() {
      Initialize();
    }
    protected AdaptiveSamplingAlgorithm(AdaptiveSamplingAlgorithm original, Cloner cloner) : base(original, cloner) {
      singleObjectiveQualityAnalyzer = cloner.Clone(original.singleObjectiveQualityAnalyzer);
      evaluationsTerminator = cloner.Clone(original.evaluationsTerminator);
      qualityTerminator = cloner.Clone(original.qualityTerminator);
      executionTimeTerminator = cloner.Clone(original.executionTimeTerminator);
      sampleCollector = cloner.Clone(original.sampleCollector);
      infillSolver = cloner.Clone(original.infillSolver);
      infillSolutionsCreator = cloner.Clone(original.infillSolutionsCreator);
      correlationAnalyzer = cloner.Clone(original.correlationAnalyzer);
      sampleAnalyzer = cloner.Clone(original.sampleAnalyzer);
      modelAnalyzer = cloner.Clone(original.modelAnalyzer);
      fitnessClusterAnalyzer = cloner.Clone(original.fitnessClusterAnalyzer);
      Initialize();
    }
    public override IDeepCloneable Clone(Cloner cloner) {
      return new AdaptiveSamplingAlgorithm(this, cloner);
    }
    public AdaptiveSamplingAlgorithm() {

      #region Add parameters
      var criteria = new ItemSet<IInfillCriterion> {
        new AugmentedExpectedImprovement(),
        new ExpectedImprovement(),
        new ExpectedQuality(),
        new ExpectedQuantileImprovement(),
        new MinimalQuantileCriterium(),
        new NeighbourDistance(),
        new PluginExpectedImprovement()
      };
      var cmaes = new CMAEvolutionStrategy.CMAEvolutionStrategy {
        MaximumGenerations = 300,
        PopulationSize = 50,
        Problem = new InfillProblem()
      };
      var model = new GaussianProcessRegression {
        Problem = new RegressionProblem()
      };
      model.CovarianceFunctionParameter.Value = new CovarianceRationalQuadraticIso();

      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
      Parameters.Add(new FixedValueParameter<MultiTerminator>("Terminator", "The termination criteria that defines if the algorithm should continue or stop.", new MultiTerminator()));
      Parameters.Add(new FixedValueParameter<MultiAnalyzer>(AnalyzerParameterName, "The operator used to analyze the solutions each iteration.", new MultiAnalyzer()));
      Parameters.Add(new FixedValueParameter<IntValue>(InitialEvaluationsParameterName, "The number of random solutions that are evaluated (in parallel) per iteration.", new IntValue(100)));
      Parameters.Add(new ConstrainedValueParameter<IInfillCriterion>(InfillCriterionParameterName, "Decision what value should decide the next sample", criteria, criteria.OfType<ExpectedImprovement>().Single()));
      Parameters.Add(new ValueParameter<IDataAnalysisAlgorithm<IRegressionProblem>>(RegressionAlgorithmParameterName, "The model used to approximate the problem", model));
      Parameters.Add(new ValueParameter<Algorithm>(InfillOptimizationAlgorithmParameterName, "The algorithm used to solve the expected improvement subproblem", cmaes));
      Parameters.Add(new FixedValueParameter<BoolValue>(RemoveDuplicatesParamterName, "Whether (almost) duplicate points in the dataset should be perturbed before evaluation (this greatly increases numerical stability for some model types)", new BoolValue(true)));
      Parameters.Add(new FixedValueParameter<IntValue>("Maximal Model Size", "The maximum number of sample points used to build the model", new IntValue(-1)));
      #endregion

      #region Create operators
      var randomCreator = new RandomCreator();
      var variableCreator = new VariableCreator { Name = "Initialize Variables" };
      var resultsCollector = new ResultsCollector();
      var solutionCreator = new SolutionsCreator { Name = "Create Initial Solutions" };
      var analyzerPlaceholder = new Placeholder { Name = "Analyzer (Placeholder)" };
      var evaluationsCounter = new IntCounter { Name = "Increment EvaluatedSolutions" };
      var solutionsCollectorProcessor = new UniformSubScopesProcessor { Name = "Collect Samples to Dataset" };
      var phaseChecker = new ConditionalBranch { Name = "Inital Samples evaluated?" };
      var initialPhaseChecker = new Comparator { Name = "Compare Initial Samples to evaluated Solutions" };
      var modelbuilder = new ModelBuilder { Name = "Build Model" };
      sampleCollector = new SampleCollector { Name = "Collect Sample" };
      infillSolutionsCreator = new SolutionsCreator { Name = "Create Adaptive Solutions" };
      infillSolver = new InfillSolver();
      var subScopesRemover = new SubScopesRemover();
      var iterationsCounter = new IntCounter { Name = "Increment Iterations" };
      var terminationOperator = new TerminationOperator();
      #endregion

      #region Wire operators
      const string iterationsVariableName = "Iterations";
      const string evaluatedSolutionsVariableName = "EvaluatedSolutions";
      const string datasetVariableName = "SampleSet";
      const string modelVariableName = "Model";
      const string initialPhaseVariableName = "InitialPhase";
      const string infillBoundsVariableName = "InfillBounds";

      OperatorGraph.InitialOperator = randomCreator;

      randomCreator.SeedParameter.Value = null;
      randomCreator.SeedParameter.ActualName = SeedParameter.Name;
      randomCreator.SetSeedRandomlyParameter.Value = null;
      randomCreator.SetSeedRandomlyParameter.ActualName = SetSeedRandomlyParameter.Name;
      randomCreator.Successor = variableCreator;

      variableCreator.CollectedValues.Add(new ValueParameter<IntValue>(iterationsVariableName, new IntValue(0)));
      variableCreator.CollectedValues.Add(new ValueParameter<IntValue>(evaluatedSolutionsVariableName, new IntValue(0)));
      variableCreator.CollectedValues.Add(new ValueParameter<ModifiableDataset>(datasetVariableName, new ModifiableDataset()));
      variableCreator.CollectedValues.Add(new ValueParameter<IRegressionSolution>(modelVariableName, "The regression solution representing the model for EGO", null));
      variableCreator.CollectedValues.Add(new FixedValueParameter<BoolValue>(initialPhaseVariableName, new BoolValue(false)));
      variableCreator.CollectedValues.Add(new ValueParameter<DoubleMatrix>(infillBoundsVariableName));
      variableCreator.Successor = resultsCollector;

      resultsCollector.CollectedValues.Add(new LookupParameter<IntValue>(iterationsVariableName, "The current iteration number."));
      resultsCollector.CollectedValues.Add(new LookupParameter<IntValue>(evaluatedSolutionsVariableName, "The current number of evaluated solutions."));
      resultsCollector.CollectedValues.Add(new LookupParameter<ModifiableDataset>(datasetVariableName, "A dataset containing all evaluated solutions"));
      resultsCollector.Successor = phaseChecker;

      phaseChecker.ConditionParameter.ActualName = initialPhaseVariableName;
      phaseChecker.FalseBranch = solutionCreator;
      phaseChecker.TrueBranch = modelbuilder;

      #region Adaptive Phase
      modelbuilder.DatasetParameter.ActualName = datasetVariableName;
      modelbuilder.ModelParameter.ActualName = modelVariableName;
      modelbuilder.RegressionAlgorithmParameter.ActualName = RegressionAlgorithmParameter.Name;
      ParameterizeStochasticOperator(modelbuilder);
      modelbuilder.Successor = infillSolutionsCreator;
      modelbuilder.MaxModelSizeParameter.ActualName = MaxModelSizeParameter.Name;

      infillSolutionsCreator.SolutionCreatorParameter.Value = infillSolver;
      infillSolutionsCreator.Successor = evaluationsCounter;
      infillSolutionsCreator.NumberOfSolutionsParameter.Value = new IntValue(1);

      infillSolver.InfillOptimizationAlgorithmParamter.ActualName = InfillOptimizationAlgorithmParameterName;
      infillSolver.ModelParameter.ActualName = modelVariableName;
      infillSolver.RemoveDuplicatesParameter.ActualName = RemoveDuplicatesParameter.Name;
      infillSolver.InfillBoundsParameter.ActualName = infillBoundsVariableName;
      #endregion

      #region Initial Phase 
      solutionCreator.NumberOfSolutionsParameter.Value = new IntValue(1);
      solutionCreator.ParallelParameter.Value.Value = true;
      solutionCreator.Successor = evaluationsCounter;
      #endregion

      evaluationsCounter.ValueParameter.ActualName = evaluatedSolutionsVariableName;
      evaluationsCounter.Increment = null;
      evaluationsCounter.IncrementParameter.Value = new IntValue(1);
      evaluationsCounter.Successor = initialPhaseChecker;

      initialPhaseChecker.Comparison = new Comparison(ComparisonType.GreaterOrEqual);
      initialPhaseChecker.LeftSideParameter.ActualName = evaluatedSolutionsVariableName;
      initialPhaseChecker.RightSideParameter.ActualName = InitialEvaluationsParameterName;
      initialPhaseChecker.ResultParameter.ActualName = initialPhaseVariableName;
      initialPhaseChecker.Successor = analyzerPlaceholder;

      analyzerPlaceholder.OperatorParameter.ActualName = AnalyzerParameter.Name;
      analyzerPlaceholder.Successor = solutionsCollectorProcessor;

      solutionsCollectorProcessor.Operator = sampleCollector;
      solutionsCollectorProcessor.Depth.Value = 1;
      solutionsCollectorProcessor.Successor = subScopesRemover;

      sampleCollector.DatasetParameter.ActualName = datasetVariableName;

      subScopesRemover.RemoveAllSubScopes = true;
      subScopesRemover.Successor = iterationsCounter;

      iterationsCounter.ValueParameter.ActualName = "Iterations";
      iterationsCounter.Increment = new IntValue(1);
      iterationsCounter.Successor = terminationOperator;



      terminationOperator.TerminatorParameter.ActualName = TerminatorParameter.Name;
      terminationOperator.ContinueBranch = phaseChecker;

      #endregion

      #region Create analyzers
      singleObjectiveQualityAnalyzer = new BestAverageWorstQualityAnalyzer();
      correlationAnalyzer = new CorrelationAnalyzer();
      correlationAnalyzer.ModelParameter.ActualName = modelVariableName;

      sampleAnalyzer = new EvaluatedSolutionsAnalyzer();
      sampleAnalyzer.PhaseParameter.ActualName = initialPhaseVariableName;

      modelAnalyzer = new ModelQualityAnalyzer();
      modelAnalyzer.ModelParameter.ActualName = modelVariableName;

      variabilityAnalyzer = new VariableVariabilityAnalyzer();
      variabilityAnalyzer.DatasetParameter.ActualName = datasetVariableName;
      variabilityAnalyzer.InitialEvaluationsParameter.ActualName = InitialEvaluationsParameter.Name;

      fitnessClusterAnalyzer = new FitnessClusteringAnalyzer();
      fitnessClusterAnalyzer.DatasetParameter.ActualName = datasetVariableName;
      #endregion

      #region Create terminators

      evaluationsTerminator = new ComparisonTerminator<IntValue>("EvaluatedSolutions", ComparisonType.Less,
        new IntValue(1000)) { Name = "Evaluated solutions." };
      qualityTerminator = new SingleObjectiveQualityTerminator { Name = "Quality" };
      executionTimeTerminator = new ExecutionTimeTerminator(this, new TimeSpanValue(TimeSpan.FromMinutes(5)));

      #endregion

      #region Parameterize
      UpdateAnalyzers();
      ParameterizeAnalyzers();
      UpdateTerminators();
      #endregion

      Initialize();
    }
    #endregion

    private void Initialize() {
      if (SingleObjectiveProblem != null) {
        SingleObjectiveProblem.Evaluator.QualityParameter.ActualNameChanged +=
         Evaluator_QualityParameter_ActualNameChanged;
      }

      singleObjectiveQualityAnalyzer.CurrentBestQualityParameter.NameChanged +=
        QualityAnalyzer_CurrentBestQualityParameter_NameChanged;
      RegisterParameterEvents();
    }

    #region Parameterization
    private void ParameterizeTerminators() {
      if (SingleObjectiveProblem != null)
        qualityTerminator.Parameterize(singleObjectiveQualityAnalyzer.CurrentBestQualityParameter,
          SingleObjectiveProblem);
    }
    private void ParameterizeAnalyzers() {
      //TODO how to guarantee ResultsParameters and QualityParameters are really there? This seems hardcoded
      singleObjectiveQualityAnalyzer.ResultsParameter.ActualName = "Results";
      singleObjectiveQualityAnalyzer.ResultsParameter.Hidden = true;
      singleObjectiveQualityAnalyzer.QualityParameter.Depth = 1;
      correlationAnalyzer.ResultsParameter.ActualName = "Results";
      correlationAnalyzer.ResultsParameter.Hidden = true;
      correlationAnalyzer.QualityParameter.Depth = 1;
      correlationAnalyzer.RealVectorParameter.Depth = 1;
      sampleAnalyzer.ResultsParameter.ActualName = "Results";
      sampleAnalyzer.ResultsParameter.Hidden = true;
      sampleAnalyzer.QualityParameter.Depth = 1;
      sampleAnalyzer.RealVectorParameter.Depth = 1;
      modelAnalyzer.ResultsParameter.ActualName = "Results";
      modelAnalyzer.ResultsParameter.Hidden = true;
      fitnessClusterAnalyzer.ResultsParameter.ActualName = "Results";
      fitnessClusterAnalyzer.ResultsParameter.Hidden = true;


      if (SingleObjectiveProblem == null) return;
      singleObjectiveQualityAnalyzer.MaximizationParameter.ActualName = SingleObjectiveProblem.MaximizationParameter.Name;
      singleObjectiveQualityAnalyzer.MaximizationParameter.Hidden = true;
      singleObjectiveQualityAnalyzer.QualityParameter.ActualName = SingleObjectiveProblem.Evaluator.QualityParameter.ActualName;
      singleObjectiveQualityAnalyzer.QualityParameter.Hidden = true;
      singleObjectiveQualityAnalyzer.BestKnownQualityParameter.ActualName = SingleObjectiveProblem.BestKnownQualityParameter.Name;
      singleObjectiveQualityAnalyzer.BestKnownQualityParameter.Hidden = true;
      correlationAnalyzer.QualityParameter.ActualName = SingleObjectiveProblem.Evaluator.QualityParameter.ActualName;
      correlationAnalyzer.QualityParameter.Hidden = true;
      sampleAnalyzer.QualityParameter.ActualName = SingleObjectiveProblem.Evaluator.QualityParameter.ActualName;
      sampleAnalyzer.QualityParameter.Hidden = true;
    }
    private void ParameterizeSolutionsCreators() {
      SolutionsCreator.EvaluatorParameter.ActualName = Problem.EvaluatorParameter.Name;
      SolutionsCreator.SolutionCreatorParameter.ActualName = Problem.SolutionCreatorParameter.Name;
      infillSolutionsCreator.EvaluatorParameter.ActualName = Problem.EvaluatorParameter.Name;
    }
    private void ParameterizeStochasticOperator(IOperator @operator) {
      var stochasticOperator = @operator as IStochasticOperator;
      if (stochasticOperator == null) return;
      stochasticOperator.RandomParameter.ActualName = "Random";
      stochasticOperator.RandomParameter.Hidden = true;
    }
    private void ParameterizeProblemSpecificOperators() {
      if (SingleObjectiveProblem == null) throw new ArgumentException("Adaptive sampling requires a single objective problem");
      infillSolver.MaximizationParameter.ActualName = SingleObjectiveProblem.MaximizationParameter.Name;
      var rcreator = Problem.SolutionCreator as IRealVectorCreator;
      if (rcreator == null) throw new ArgumentException("Adaptive sampling requires real vectors as solution candidates");
      infillSolver.LengthParameter.ActualName = rcreator.LengthParameter.ActualName;
      infillSolver.BoundsParameter.ActualName = rcreator.BoundsParameter.ActualName;
      // if (rcreator.LengthParameter.Value != null)
      infillSolver.LengthParameter.Value = rcreator.LengthParameter.Value;
      // if (rcreator.BoundsParameter.Value != null)
      infillSolver.BoundsParameter.Value = rcreator.BoundsParameter.Value;
      infillSolver.RealVectorParameter.ActualName = rcreator.RealVectorParameter.ActualName;

      sampleCollector.RealVectorParameter.ActualName = rcreator.RealVectorParameter.ActualName;
      sampleCollector.QualityParameter.ActualName = SingleObjectiveProblem.Evaluator.QualityParameter.ActualName;

      correlationAnalyzer.RealVectorParameter.ActualName = rcreator.RealVectorParameter.ActualName;
      sampleAnalyzer.RealVectorParameter.ActualName = rcreator.RealVectorParameter.ActualName;
    }
    #endregion

    public override void Start() {
      ParameterizeProblemSpecificOperators();
      ((InfillProblem)InfillOptimizationAlgorithm.Problem).InfillCriterion = InfillCriterion;
      base.Start();
    }
    public override void Stop() {
      if (InfillOptimizationAlgorithm.ExecutionState == ExecutionState.Started || InfillOptimizationAlgorithm.ExecutionState == ExecutionState.Paused)
        InfillOptimizationAlgorithm.Stop();
      if (RegressionAlgorithm.ExecutionState == ExecutionState.Started || RegressionAlgorithm.ExecutionState == ExecutionState.Paused)
        RegressionAlgorithm.Stop();
      base.Stop();
    }
    public override void Pause() {
      if (InfillOptimizationAlgorithm.ExecutionState == ExecutionState.Started) InfillOptimizationAlgorithm.Pause();
      if (RegressionAlgorithm.ExecutionState == ExecutionState.Started) RegressionAlgorithm.Pause();
      base.Pause();
    }

    #region events
    private void RegisterParameterEvents() {
      InfillOptimizationAlgorithmParameter.ValueChanged += OnInfillAlgorithmChanged;
      InfillOptimizationAlgorithm.ProblemChanged += OnInfillProblemChanged;
      InfillCriterionParameter.ValueChanged += OnInfillCriterionChanged;
    }
    private void OnInfillCriterionChanged(object sender, EventArgs e) {
      ((InfillProblem)InfillOptimizationAlgorithm.Problem).InfillCriterion = InfillCriterion;
    }
    private void OnInfillAlgorithmChanged(object sender, EventArgs e) {
      InfillOptimizationAlgorithm.Problem = new InfillProblem { InfillCriterion = InfillCriterion };
      InfillOptimizationAlgorithm.ProblemChanged -= OnInfillProblemChanged; //avoid double attaching
      InfillOptimizationAlgorithm.ProblemChanged += OnInfillProblemChanged;
    }
    private void OnInfillProblemChanged(object sender, EventArgs e) {
      //enforce an infill problem for the infill optimization algorithm
      InfillOptimizationAlgorithm.ProblemChanged -= OnInfillProblemChanged;
      InfillOptimizationAlgorithm.Problem = new InfillProblem { InfillCriterion = InfillCriterion };
      InfillOptimizationAlgorithm.ProblemChanged += OnInfillProblemChanged;
    }
    protected override void OnProblemChanged() {
      base.OnProblemChanged();
      ParameterizeStochasticOperator(Problem.SolutionCreator);
      foreach (var @operator in Problem.Operators.OfType<IOperator>())
        ParameterizeStochasticOperator(@operator);

      //TODO ParameterizeIterationBasedOperators();

      ParameterizeSolutionsCreators();
      ParameterizeAnalyzers();
      ParameterizeTerminators();

      if (SingleObjectiveProblem != null)
        SingleObjectiveProblem.Evaluator.QualityParameter.ActualNameChanged +=
          Evaluator_QualityParameter_ActualNameChanged;

      UpdateAnalyzers();
      UpdateTerminators();
    }
    protected override void RegisterProblemEvents() {
      base.RegisterProblemEvents();
      var maximizationParameter = (IValueParameter<BoolValue>)SingleObjectiveProblem?.MaximizationParameter;
      if (maximizationParameter != null) maximizationParameter.ValueChanged += MaximizationParameter_ValueChanged;

    }
    protected override void DeregisterProblemEvents() {
      var maximizationParameter = (IValueParameter<BoolValue>)SingleObjectiveProblem?.MaximizationParameter;
      if (maximizationParameter != null) maximizationParameter.ValueChanged -= MaximizationParameter_ValueChanged;
      base.DeregisterProblemEvents();
    }
    protected override void Problem_SolutionCreatorChanged(object sender, EventArgs e) {
      base.Problem_SolutionCreatorChanged(sender, e);
      ParameterizeStochasticOperator(Problem.SolutionCreator);

      if (SingleObjectiveProblem != null) SingleObjectiveProblem.Evaluator.QualityParameter.ActualNameChanged += Evaluator_QualityParameter_ActualNameChanged;

      ParameterizeSolutionsCreators();
      ParameterizeAnalyzers();
    }
    protected override void Problem_EvaluatorChanged(object sender, EventArgs e) {
      base.Problem_EvaluatorChanged(sender, e);
      foreach (var @operator in Problem.Operators.OfType<IOperator>())
        ParameterizeStochasticOperator(@operator);

      UpdateAnalyzers();

      ParameterizeSolutionsCreators();
    }
    protected override void Problem_OperatorsChanged(object sender, EventArgs e) {
      base.Problem_OperatorsChanged(sender, e);
      //TODO ParameterizeIterationBasedOperators();
      UpdateTerminators();
    }
    private void MaximizationParameter_ValueChanged(object sender, EventArgs e) {
      ParameterizeTerminators();
    }
    private void Evaluator_QualityParameter_ActualNameChanged(object sender, EventArgs e) {
      ParameterizeAnalyzers();
    }
    private void QualityAnalyzer_CurrentBestQualityParameter_NameChanged(object sender, EventArgs e) {
      ParameterizeTerminators();
    }
    #endregion

    #region Updates
    private void UpdateAnalyzers() {
      Analyzer.Operators.Clear();
      if (Problem == null) return;
      if (SingleObjectiveProblem != null) {
        Analyzer.Operators.Add(singleObjectiveQualityAnalyzer, singleObjectiveQualityAnalyzer.EnabledByDefault);
        Analyzer.Operators.Add(correlationAnalyzer, correlationAnalyzer.EnabledByDefault);
        Analyzer.Operators.Add(sampleAnalyzer, sampleAnalyzer.EnabledByDefault);
        Analyzer.Operators.Add(modelAnalyzer, modelAnalyzer.EnabledByDefault);
        Analyzer.Operators.Add(variabilityAnalyzer, variabilityAnalyzer.EnabledByDefault);
        Analyzer.Operators.Add(fitnessClusterAnalyzer, fitnessClusterAnalyzer.EnabledByDefault);
      }
      foreach (var analyzer in Problem.Operators.OfType<IAnalyzer>())
        Analyzer.Operators.Add(analyzer, analyzer.EnabledByDefault);
    }
    private void UpdateTerminators() {
      var newTerminators = new Dictionary<ITerminator, bool> {
        {
          evaluationsTerminator,
          !Terminator.Operators.Contains(evaluationsTerminator) ||
          Terminator.Operators.ItemChecked(evaluationsTerminator)
        }, {
          executionTimeTerminator,
          Terminator.Operators.Contains(executionTimeTerminator) &&
          Terminator.Operators.ItemChecked(executionTimeTerminator)
        }
      };
      if (Problem != null) {
        if (SingleObjectiveProblem != null)
          newTerminators.Add(qualityTerminator,
            Terminator.Operators.Contains(qualityTerminator) && Terminator.Operators.ItemChecked(qualityTerminator));
        foreach (var terminator in Problem.Operators.OfType<ITerminator>())
          newTerminators.Add(terminator,
            !Terminator.Operators.Contains(terminator) || Terminator.Operators.ItemChecked(terminator));
      }

      Terminator.Operators.Clear();
      foreach (var newTerminator in newTerminators)
        Terminator.Operators.Add(newTerminator.Key, newTerminator.Value);
    }
    #endregion

  }

}