source: branches/HeuristicLab.GoalSeekingProblem/HeuristicLab.GoalSeekingProblem/3.4/MultiObjectiveGoalSeekingProblem.cs @ 14325

#region License Information
/* HeuristicLab
 * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */
#endregion

using System;
using System.Collections.Generic;
using System.Linq;
using System.Windows.Forms;
using HeuristicLab.Collections;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Encodings.RealVectorEncoding;
using HeuristicLab.Optimization;
using HeuristicLab.Parameters;
using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
using HeuristicLab.Problems.DataAnalysis;

namespace HeuristicLab.GoalSeeking {
  [Item("Goal seeking problem (multi-objective)", "Represents a single objective optimization problem which uses configurable regression models to evaluate targets from a given dataset.")]
  [Creatable("Problems")]
  [StorableClass]
  public sealed class MultiObjectiveGoalSeekingProblem : MultiObjectiveBasicProblem<RealVectorEncoding>, IGoalSeekingProblem {
    private const string ModifiableDatasetParameterName = "Dataset";
    private const string ProblemDataParameterName = "ProblemData";
    private const string ControllableParametersParameterName = "ControllableParameters";
    private const string ControllableParameterBoundsParameterName = "ControllableParameterBounds";
    private const string TargetGoalsParameterName = "TargetGoals";
    private const string TargetsParameterName = "Targets";
    private const string ModelCollectionParameterName = "ModelCollection";
    private const string RowParameterName = "Row";
    // these parameters are used by the pareto folding analyzer
    private const string QualitySumCutoffParameterName = "QualitySumCutoff";

    #region parameters
    public IFixedValueParameter<DoubleValue> QualitySumCutoffParameter {
      get { return (IFixedValueParameter<DoubleValue>)Parameters[QualitySumCutoffParameterName]; }
    }
    public IValueParameter<IRegressionProblemData> ProblemDataParameter {
      get { return (IValueParameter<IRegressionProblemData>)Parameters[ProblemDataParameterName]; }
    }
    public IValueParameter<CheckedItemList<StringValue>> ControllableParametersParameter {
      get { return (IValueParameter<CheckedItemList<StringValue>>)Parameters[ControllableParametersParameterName]; }
    }
    public IValueParameter<DoubleMatrix> ControllableParameterBoundsParameter {
      get { return (IValueParameter<DoubleMatrix>)Parameters[ControllableParameterBoundsParameterName]; }
    }
    public IFixedValueParameter<ItemCollection<IRegressionModel>> ModelCollectionParameter {
      get { return (IFixedValueParameter<ItemCollection<IRegressionModel>>)Parameters[ModelCollectionParameterName]; }
    }
    public IValueParameter<CheckedItemList<StringValue>> TargetsParameter {
      get { return (IValueParameter<CheckedItemList<StringValue>>)Parameters[TargetsParameterName]; }
    }
    public IValueParameter<DoubleMatrix> TargetGoalsParameter {
      get { return (IValueParameter<DoubleMatrix>)Parameters[TargetGoalsParameterName]; }
    }
    public IFixedValueParameter<IntValue> RowParameter {
      get { return (IFixedValueParameter<IntValue>)Parameters[RowParameterName]; }
    }
    #endregion

    #region parameter properties
    private IItemCollection<IRegressionModel> ModelCollection {
      get { return ModelCollectionParameter.Value; }
    }
    public DoubleMatrix TargetGoals {
      get { return TargetGoalsParameter.Value; }
      set { TargetGoalsParameter.Value = value; }
    }
    public double QualitySumCutoff {
      get { return QualitySumCutoffParameter.Value.Value; }
      set { QualitySumCutoffParameter.Value.Value = value; }
    }
    #endregion

    #region IProcessParameterOptimizationProblem properties
    [Storable]
    private IRegressionProblemData problemData;
    public IRegressionProblemData ProblemData {
      get { return problemData; }
      set {
        if (value == null || value == problemData) return;
        var variables = value.Dataset.DoubleVariables.ToList();
        if (Models.Any()) {
          var targets = Models.Select(x => x.TargetVariable);
          var hashset = new HashSet<string>(variables);
          foreach (var target in targets) {
            if (!hashset.Contains(target)) {
              throw new ArgumentException(string.Format("Incompatible problem data. Target \"{0}\" is missing.", target));
            }
          }
        }
        problemData = value;
        dataset = new ModifiableDataset(variables, variables.Select(x => new List<double> { ProblemData.Dataset.GetDoubleValue(x, Row) }));
        ProblemDataParameter.Value = ProblemData;
        UpdateControllableParameters();
        UpdateTargetList();
      }
    }

    public int Row {
      get { return RowParameter.Value.Value; }
      set { RowParameter.Value.Value = value; }
    }

    public IEnumerable<IRegressionModel> Models {
      get { return ModelCollectionParameter.Value; }
    }

    #region targets
    public ICheckedItemList<StringValue> TargetList {
      get { return TargetsParameter.Value; }
      set { TargetsParameter.Value = (CheckedItemList<StringValue>)value; }
    }
    // convenience method
    private IEnumerable<string> ActiveTargets {
      get { return TargetList.CheckedItems.Select(x => x.Value.Value); }
    }
    #endregion

    #region parameters
    public ICheckedItemList<StringValue> ControllableParameters {
      get { return ControllableParametersParameter.Value; }
      set { ControllableParametersParameter.Value = (CheckedItemList<StringValue>)value; }
    }
    // convenience method 
    private IEnumerable<string> ActiveParameters {
      get { return ControllableParameters.CheckedItems.Select(x => x.Value.Value); }
    }
    public DoubleMatrix ControllableParameterBounds {
      get { return ControllableParameterBoundsParameter.Value; }
      set { ControllableParameterBoundsParameter.Value = value; }
    }
    #endregion
    #endregion

    #region IProcessParameterOptimizationProblem methods
    #region models
    public IEnumerable<double> GetEstimatedGoalValues(IEnumerable<double> parameterValues, bool round = false) {
      var ds = (ModifiableDataset)dataset.Clone();
      foreach (var parameter in ActiveParameters.Zip(parameterValues, (p, v) => new { Name = p, Value = v })) {
        ds.SetVariableValue(parameter.Value, parameter.Name, 0);
      }
      var rows = new[] { 0 }; // actually just one row

      var estimatedValues =
        round ? ActiveTargets.Select(t => RoundToNearestStepMultiple(GetModels(t).Average(m => m.GetEstimatedValues(ds, rows).Single()), GetTargetStepSize(t)))
              : ActiveTargets.Select(t => GetModels(t).Average(m => m.GetEstimatedValues(ds, rows).Single()));
      return estimatedValues;
    }

    public void AddModel(IRegressionModel model) {
      var target = model.TargetVariable;
      CheckIfDatasetContainsTarget(target);
      ModelCollection.Add(model);
      OnModelsChanged(this, EventArgs.Empty);
    }

    // method which throws an exception that can be caught in the event handler if the check fails
    private void CheckIfDatasetContainsTarget(string target) {
      if (dataset.DoubleVariables.All(x => x != target))
        throw new ArgumentException(string.Format("Model target \"{0}\" does not exist in the dataset.", target));
    }

    public void RemoveModel(IRegressionModel model) {
      ModelCollection.Remove(model);
      OnModelsChanged(this, EventArgs.Empty);
    }

    public event EventHandler ModelsChanged;
    private void OnModelsChanged(object sender, EventArgs args) {
      var changed = ModelsChanged;
      if (changed == null) return;
      changed(sender, args);
    }
    #endregion

    #region targets
    public bool GetTargetActive(string target) {
      var item = TargetList.SingleOrDefault(x => x.Value == target);
      if (item == null)
        throw new ArgumentException(string.Format("SetTargetActive: Invalid target name {0}", target));
      return TargetList.ItemChecked(item);
    }

    public void SetTargetActive(string target, bool active) {
      var item = TargetList.SingleOrDefault(x => x.Value == target);
      if (item == null)
        throw new ArgumentException(string.Format("SetTargetActive: Invalid target name {0}", target));
      TargetList.SetItemCheckedState(item, active);
      OnTargetsChanged(this, EventArgs.Empty);
    }

    public double GetTargetGoal(string target) {
      if (!IsValidTarget(target))
        throw new ArgumentException(string.Format("The target variable name \"{0}\" does not exist in the dataset.", target));
      int i = TargetGoals.RowNames.TakeWhile(x => x != target).Count();
      return TargetGoals[i, 0];
    }

    public void SetTargetGoal(string target, double goal) {
      if (!IsValidTarget(target))
        throw new ArgumentException(string.Format("The target variable name \"{0}\" does not exist in the dataset.", target));
      int i = TargetGoals.RowNames.TakeWhile(x => x != target).Count();
      TargetGoals[i, 0] = goal;
      OnTargetsChanged(this, EventArgs.Empty);
    }

    public double GetTargetWeight(string target) {
      if (!IsValidTarget(target))
        throw new ArgumentException(string.Format("The target variable name \"{0}\" does not exist in the dataset.", target));
      int i = TargetGoals.RowNames.TakeWhile(x => x != target).Count();
      return TargetGoals[i, 1];
    }

    public void SetTargetWeight(string target, double weight) {
      if (!IsValidTarget(target))
        throw new ArgumentException(string.Format("The target variable name \"{0}\" does not exist in the dataset.", target));
      int i = TargetGoals.RowNames.TakeWhile(x => x != target).Count();
      TargetGoals[i, 1] = weight;
      OnTargetsChanged(this, EventArgs.Empty);
    }

    public double GetTargetVariance(string target) {
      if (!IsValidTarget(target))
        throw new ArgumentException(string.Format("The target variable name \"{0}\" does not exist in the dataset.", target));
      int i = TargetGoals.RowNames.TakeWhile(x => x != target).Count();
      return TargetGoals[i, 2];
    }

    public void SetTargetVariance(string target, double variance) {
      if (!IsValidTarget(target))
        throw new ArgumentException(string.Format("The target variable name \"{0}\" does not exist in the dataset.", target));
      int i = TargetGoals.RowNames.TakeWhile(x => x != target).Count();
      TargetGoals[i, 2] = variance;
      OnTargetsChanged(this, EventArgs.Empty);
    }

    public double GetTargetStepSize(string target) {
      if (!IsValidTarget(target))
        throw new ArgumentException(string.Format("The target variable name \"{0}\" does not exist in the dataset.", target));
      int i = TargetGoals.RowNames.TakeWhile(x => x != target).Count();
      return TargetGoals[i, 3];
    }

    public void SetTargetStepSize(string target, double stepSize) {
      if (!IsValidTarget(target))
        throw new ArgumentException(string.Format("The target variable name \"{0}\" does not exist in the dataset.", target));
      int i = TargetGoals.RowNames.TakeWhile(x => x != target).Count();
      TargetGoals[i, 3] = stepSize;
      OnTargetsChanged(this, EventArgs.Empty);
    }

    public event EventHandler TargetsChanged;
    private void OnTargetsChanged(object sender, EventArgs args) {
      var changed = TargetsChanged;
      if (changed == null) return;
      changed(sender, args);
    }
    #endregion // targets

    #region process parameters
    /// <summary>
    /// Returns the parameter bounds (min and max) and the step size for the specified parameter
    /// </summary>
    /// <param name="parameterName"></param>
    /// <returns>A double array containing the values (min, max, step) in this order</returns>
    public double[] GetParameterBounds(string parameterName) {
      var index = ControllableParameters.TakeWhile(x => x.Value != parameterName).Count();
      if (index < ControllableParameters.Count) {
        var min = ControllableParameterBounds[index, 0];
        var max = ControllableParameterBounds[index, 1];
        var step = ControllableParameterBounds[index, 2];
        return new[] { min, max, step };
      }
      throw new ArgumentException(string.Format("GetParameterBounds: Unknown parameter {0}.", parameterName));
    }

    public void SetParameterBounds(string parameterName, double min, double max, double step) {
      int i = ControllableParameterBounds.RowNames.TakeWhile(x => x != parameterName).Count();
      if (i < ControllableParameterBounds.Rows) {
        ControllableParameterBounds[i, 0] = min;
        ControllableParameterBounds[i, 1] = max;
        ControllableParameterBounds[i, 2] = step;
        UpdateEncoding();
        OnParametersChanged(this, EventArgs.Empty);
      } else {
        throw new ArgumentException(string.Format("SetParameterBounds: Invalid parameter name {0}", parameterName));
      }

    }

    public double GetParameterStepSize(string parameter) {
      int i = ControllableParameterBounds.RowNames.TakeWhile(x => x != parameter).Count();
      if (i < ControllableParameterBounds.Rows)
        return ControllableParameterBounds[i, 2];
      throw new ArgumentException(string.Format("GetParameterStepSize: Invalid parameter name {0}", parameter));
    }

    public void SetParameterStepSize(string parameter, double stepSize) {
      int i = ControllableParameterBounds.RowNames.TakeWhile(x => x != parameter).Count();
      if (i < ControllableParameterBounds.Rows) {
        ControllableParameterBounds[i, 2] = stepSize;
        OnParametersChanged(this, EventArgs.Empty);
        return;
      }
      throw new ArgumentException(string.Format("SetParameterStepSize: Invalid parameter name {0}", parameter));
    }

    public bool GetParameterActive(string parameter) {
      var item = ControllableParameters.SingleOrDefault(x => x.Value == parameter);
      if (item == null)
        throw new ArgumentException(string.Format("GetParameterActive: Invalid target name {0}", parameter));
      return ControllableParameters.ItemChecked(item);
    }

    public void SetParameterActive(string parameter, bool active) {
      var item = ControllableParameters.SingleOrDefault(x => x.Value == parameter);
      if (item == null)
        throw new ArgumentException(string.Format("SetParameterActive: Invalid target name {0}", parameter));
      ControllableParameters.SetItemCheckedState(item, active);
      OnParametersChanged(this, EventArgs.Empty);
    }

    public void SetControllableParameters(IEnumerable<string> parameterNames) {
      ControllableParameters = new CheckedItemList<StringValue>();
      foreach (var v in parameterNames) {
        ControllableParameters.Add(new StringValue(v), false);
      }
      ControllableParameters.CheckedItemsChanged += ControllableParameters_OnItemsChanged;
      ControllableParameterBounds = new DoubleMatrix(ControllableParameters.Count, 3);
      ControllableParameterBounds.RowNames = GetControllableParameters();
      ControllableParameterBounds.ColumnNames = new[] { "Min", "Max", "Step" };

      for (int i = 0; i < ControllableParameters.Count; ++i) {
        var itemName = ControllableParameters[i].Value;
        var values = ProblemData.Dataset.GetReadOnlyDoubleValues(itemName).Where(x => !double.IsNaN(x) && !double.IsInfinity(x)).ToList();
        if (!values.Any()) continue;

        // add a 20% margin to allow the optimization algorithm more freedom of exploration
        ControllableParameterBounds[i, 0] = 0.8 * values.Min(); // min
        ControllableParameterBounds[i, 1] = 1.2 * values.Max(); // max
        ControllableParameterBounds[i, 2] = 1e-6;               // step
      }
      OnParametersChanged(this, EventArgs.Empty);
    }

    public IEnumerable<string> GetControllableParameters() {
      return ControllableParameters.Select(x => x.Value);
    }

    public event EventHandler ParametersChanged;
    private void OnParametersChanged(object sender, EventArgs args) {
      var changed = ParametersChanged;
      if (changed == null) return;
      changed(sender, args);
    }
    #endregion // process parameters
    #endregion // IGoalSeekingProblem methods

    #region data members
    [Storable]
    private ModifiableDataset dataset; // modifiable dataset

    [Storable]
    private bool[] maximization;
    public override bool[] Maximization {
      get { return maximization ?? new bool[] { false }; }
    }

    public ValueParameter<BoolArray> MaximizationParameter {
      get { return (ValueParameter<BoolArray>)Parameters["Maximization"]; }
    }
    #endregion

    #region constructors
    [StorableConstructor]
    private MultiObjectiveGoalSeekingProblem(bool deserializing) : base(deserializing) { }

    private MultiObjectiveGoalSeekingProblem(MultiObjectiveGoalSeekingProblem original, Cloner cloner) : base(original, cloner) {
      this.dataset = cloner.Clone(original.dataset);
      this.problemData = cloner.Clone(original.problemData);

      RegisterEvents();
    }

    public override IDeepCloneable Clone(Cloner cloner) {
      return new MultiObjectiveGoalSeekingProblem(this, cloner);
    }

    [StorableHook(HookType.AfterDeserialization)]
    private void AfterDeserialization() {
      if (Parameters.ContainsKey("Accuracy"))
        Parameters.Remove("Accuracy");

      if (!Parameters.ContainsKey(QualitySumCutoffParameterName)) {
        Parameters.Add(new FixedValueParameter<DoubleValue>(QualitySumCutoffParameterName, new DoubleValue(0.2)));
        QualitySumCutoffParameter.Hidden = true;
      }

      if (ProblemData == null && Parameters.ContainsKey(ProblemDataParameterName)) {
        ProblemData = ProblemDataParameter.Value;
      }

      if (!Parameters.ContainsKey(ModifiableDatasetParameterName)) {
        Parameters.Add(new ValueParameter<IDataset>(ModifiableDatasetParameterName, dataset) { Hidden = true });
      }

      // backwards-compatibility
      if (Parameters.ContainsKey("Models")) {
        var solutions = ((IFixedValueParameter<ItemCollection<IRegressionSolution>>)Parameters["Models"]).Value;
        var models = new ItemCollection<IRegressionModel>();
        foreach (var solution in solutions) {
          var model = solution.Model;
          model.TargetVariable = solution.ProblemData.TargetVariable;
          models.Add(model);
        }
        if (Parameters.ContainsKey(ModelCollectionParameterName))
          Parameters.Remove(ModelCollectionParameterName);
        Parameters.Add(new FixedValueParameter<ItemCollection<IRegressionModel>>(ModelCollectionParameterName, models));
      }

      RegisterEvents();
    }

    public MultiObjectiveGoalSeekingProblem() {
      Parameters.Add(new ValueParameter<IRegressionProblemData>(ProblemDataParameterName, new RegressionProblemData()));
      Parameters.Add(new ValueParameter<IDataset>(ModifiableDatasetParameterName, dataset) { Hidden = true });
      Parameters.Add(new ValueParameter<CheckedItemList<StringValue>>(ControllableParametersParameterName));
      Parameters.Add(new ValueParameter<CheckedItemList<StringValue>>(TargetsParameterName));
      Parameters.Add(new ValueParameter<DoubleMatrix>(ControllableParameterBoundsParameterName));
      Parameters.Add(new FixedValueParameter<ItemCollection<IRegressionModel>>(ModelCollectionParameterName, new ItemCollection<IRegressionModel>()));
      Parameters.Add(new ValueParameter<DoubleMatrix>(TargetGoalsParameterName)); // model target weights
      Parameters.Add(new FixedValueParameter<IntValue>(RowParameterName));
      Parameters.Add(new FixedValueParameter<DoubleValue>(QualitySumCutoffParameterName, new DoubleValue(0.2)));

      QualitySumCutoffParameter.Hidden = true;

      // when the problem is created, the problem data parameter will be set to a default value
      // set the internal property to the same value
      ProblemData = ProblemDataParameter.Value;

      UpdateControllableParameters();
      UpdateTargetList();
      RegisterEvents();
    }
    #endregion

    public override double[] Evaluate(Individual individual, IRandom random) {
      var vector = individual.RealVector();
      vector.ElementNames = ActiveParameters;

      int i = 0;
      // round vector according to parameter step sizes
      foreach (var parameter in ControllableParameters.CheckedItems) {
        var step = ControllableParameterBounds[parameter.Index, 2];
        vector[i] = RoundToNearestStepMultiple(vector[i], step);
        ++i;
      }
      var estimatedValues = GetEstimatedGoalValues(vector, round: true);
      var qualities = TargetList.CheckedItems.Zip(estimatedValues, (t, v) => new { Name = t.Value.Value, Index = t.Index, EstimatedValue = v })
                                .Select(target => {
                                  var goal = TargetGoals[target.Index, 0];
                                  var weight = TargetGoals[target.Index, 1];
                                  var variance = TargetGoals[target.Index, 2];
                                  return weight * Math.Pow(target.EstimatedValue - goal, 2) / variance;
                                });
      return qualities.ToArray();
    }

    public override void Analyze(Individual[] individuals, double[][] qualities, ResultCollection results, IRandom random) {
      var matrix = FilterFrontsByQualitySum(individuals, qualities, Math.Max(QualitySumCutoff, qualities.Min(x => x.Sum())));
      const string resultName = "Pareto Front Solutions"; // disclaimer: not really a pareto front
      if (!results.ContainsKey(resultName)) {
        results.Add(new Result(resultName, matrix));
      } else {
        results[resultName].Value = matrix;
      }
      base.Analyze(individuals, qualities, results, random);
    }

    private DoubleMatrix FilterFrontsByQualitySum(Individual[] individuals, double[][] qualities, double qualitySumCutoff) {
      var activeParameters = ActiveParameters.ToList();
      var activeTargets = ActiveTargets.ToList();
      var filteredModels = new List<double[]>();
      var rowNames = new List<string>();
      // build list of column names by combining target and parameter names (with their respective original and estimated values)
      var columnNames = new List<string> { "Quality Sum" };
      foreach (var target in activeTargets) {
        columnNames.Add(target);
        columnNames.Add(target + " (estimated)");
      }
      foreach (var controllableParameter in activeParameters) {
        columnNames.Add(controllableParameter);
        columnNames.Add(controllableParameter + " (estimated)");
        columnNames.Add(controllableParameter + " (deviation)");
      }
      // filter models based on their quality sum; remove duplicate models
      var dec = new DoubleEqualityComparer(); // comparer which uses the IsAlmost method for comparing floating point numbers
      for (int i = 0; i < individuals.Length; ++i) {
        var qualitySum = qualities[i].Sum();
        if (qualitySum > qualitySumCutoff)
          continue;
        var vector = individuals[i].RealVector();
        var estimatedValues = GetEstimatedGoalValues(vector).ToList();
        var rowValues = new double[columnNames.Count];
        rowValues[0] = qualitySum;
        int offset = 1;
        for (int j = 0; j < activeTargets.Count * 2; j += 2) {
          int k = j + offset;
          var goal = GetTargetGoal(activeTargets[j / 2]);
          rowValues[k] = goal; // original value 
          rowValues[k + 1] = estimatedValues[j / 2]; // estimated value
        }
        offset += activeTargets.Count * 2;
        for (int j = 0; j < activeParameters.Count * 3; j += 3) {
          int k = j + offset;
          rowValues[k] = problemData.Dataset.GetDoubleValue(columnNames[k], Row);
          rowValues[k + 1] = vector[j / 3];
          rowValues[k + 2] = rowValues[k + 1] - rowValues[k];
        }
        if (!filteredModels.Any(x => x.SequenceEqual(rowValues, dec))) {
          rowNames.Add((i + 1).ToString());
          filteredModels.Add(rowValues);
        }
      }
      var matrix = new DoubleMatrix(filteredModels.Count, columnNames.Count) { RowNames = rowNames, ColumnNames = columnNames, SortableView = true };
      for (int i = 0; i < filteredModels.Count; ++i) {
        for (int j = 0; j < filteredModels[i].Length; ++j) {
          matrix[i, j] = filteredModels[i][j];
        }
      }
      return matrix;
    }

    #region event handlers
    private void RegisterEvents() {
      ProblemDataParameter.ValueChanged += OnProblemDataChanged;
      ModelCollectionParameter.Value.ItemsAdded += ModelCollection_OnItemsAdded;
      ModelCollectionParameter.Value.ItemsRemoved += ModelCollection_OnItemsRemoved;
      RowParameter.Value.ValueChanged += OnRowChanged;
      ControllableParameters.CheckedItemsChanged += ControllableParameters_OnItemsChanged;
      ControllableParameterBounds.ItemChanged += ControllableParameterBounds_ItemChanged;
    }

    private void OnRowChanged(object o, EventArgs e) {
      // set variables in the modifiable dataset according to the new row
      foreach (var v in dataset.DoubleVariables)
        dataset.SetVariableValue(ProblemData.Dataset.GetDoubleValue(v, Row), v, 0);
      // set the correct targets
      UpdateTargetList();
    }

    private void OnProblemDataChanged(object o, EventArgs e) {
      try {
        ProblemData = ProblemDataParameter.Value;
      }
      catch (ArgumentException exception) {
        MessageBox.Show(exception.Message, "Update Problem Data", MessageBoxButtons.OK, MessageBoxIcon.Error);
        ProblemDataParameter.Value = problemData;
      }
    }

    private void ModelCollection_OnItemsAdded(object sender, CollectionItemsChangedEventArgs<IRegressionModel> e) {
      if (e.Items == null) return;

      var collection = (IObservableCollection<IRegressionModel>)sender;
      var newItems = e.Items.ToList();

      foreach (var model in e.Items) {
        try {
          CheckIfDatasetContainsTarget(model.TargetVariable);
        }
        catch (ArgumentException exception) {
          MessageBox.Show(exception.Message, "Add Model", MessageBoxButtons.OK, MessageBoxIcon.Error);
          newItems.Remove(model);
          collection.Remove(model);
        }
      }
      UpdateTargetList();
      OnModelsChanged(this, EventArgs.Empty);
    }

    private void ModelCollection_OnItemsRemoved(object sender, CollectionItemsChangedEventArgs<IRegressionModel> e) {
      if (e.Items == null) return;
      UpdateTargetList();
      OnModelsChanged(this, EventArgs.Empty);
    }

    private void ControllableParameters_OnItemsChanged(object o, CollectionItemsChangedEventArgs<IndexedItem<StringValue>> e) {
      UpdateEncoding();
    }

    private void ControllableParameterBounds_ItemChanged(object o, EventArgs e) {
      UpdateEncoding();
    }
    #endregion

    #region helper methods
    private void UpdateControllableParameters() {
      if (ProblemData == null) return;
      var variablesUsedForPrediction = ModelCollection.Any()
        ? ModelCollection.SelectMany(x => x.VariablesUsedForPrediction).Distinct()
        : ProblemData.Dataset.DoubleVariables;
      SetControllableParameters(variablesUsedForPrediction);
    }

    private void UpdateTargetList() {
      if (ProblemData == null) return;
      if (!Models.Any()) {
        TargetGoals = new DoubleMatrix();
        maximization = new[] { false };
        MaximizationParameter.Value = (BoolArray)new BoolArray(maximization).AsReadOnly();
        TargetList = new CheckedItemList<StringValue>();
        return;
      }

      var targetNames = Models.Select(x => x.TargetVariable).Distinct().ToList();
      var oldTargetGoals = (DoubleMatrix)TargetGoals.Clone();
      var oldRowIndices = oldTargetGoals.RowNames.Select((x, i) => new { x, i }).ToDictionary(x => x.x, x => x.i);
      TargetGoals = new DoubleMatrix(targetNames.Count, 4);
      TargetGoals.RowNames = targetNames;
      TargetGoals.ColumnNames = new[] { "Goal", "Weight", "Variance", "Step size" };

      TargetList = new CheckedItemList<StringValue>();
      for (int i = 0; i < targetNames.Count; ++i) {
        TargetList.Add(new StringValue(targetNames[i]), true);
        int rowIndex;
        if (oldRowIndices.TryGetValue(targetNames[i], out rowIndex)) {
          for (int j = 0; j < TargetGoals.Columns; ++j)
            TargetGoals[i, j] = oldTargetGoals[rowIndex, j];
        } else {
          TargetGoals[i, 0] = ProblemData.Dataset.GetDoubleValue(targetNames[i], Row);
          TargetGoals[i, 1] = 1.0;
          TargetGoals[i, 2] = ProblemData.Dataset.GetReadOnlyDoubleValues(targetNames[i]).Variance();
          TargetGoals[i, 3] = 1e-6;
        }
      }
      maximization = new bool[targetNames.Count];
      MaximizationParameter.Value = (BoolArray)new BoolArray(maximization).AsReadOnly();
    }

    private void UpdateEncoding() {
      var activeParameters = ActiveParameters.ToList();
      if (Encoding == null)
        Encoding = new RealVectorEncoding(activeParameters.Count);
      else
        Encoding.Length = activeParameters.Count;

      Encoding.Bounds = new DoubleMatrix(activeParameters.Count, 2); // only two columns: min and max
      Encoding.Bounds.RowNames = activeParameters;
      Encoding.Bounds.ColumnNames = new[] { "Min.", "Max." };

      int i = 0;
      foreach (var item in ControllableParameters.CheckedItems) {
        var index = item.Index;
        Encoding.Bounds[i, 0] = ControllableParameterBounds[index, 0];
        Encoding.Bounds[i, 1] = ControllableParameterBounds[index, 1];
        ++i;
      }
    }

    private bool IsValidTarget(string target) {
      return TargetList.Any(x => x.Value == target);
    }
    private static double RoundToNearestStepMultiple(double value, double step) {
      return step * (long)Math.Round(value / step);
    }
    private IEnumerable<IRegressionModel> GetModels(string target) {
      return ModelCollection.Where(x => x.TargetVariable == target);
    }
    private class DoubleEqualityComparer : IEqualityComparer<double> {
      public bool Equals(double x, double y) { return x.IsAlmost(y); }
      public int GetHashCode(double obj) { return obj.GetHashCode(); }
    }
    #endregion
  }
}