source: branches/2988_ModelsOfModels2/HeuristicLab.Algorithms.EMM/EMMAlgorithm/EMMBaseAlgorithm.cs @ 17134

#region License Information
/* HeuristicLab
 * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */
#endregion

using HEAL.Attic;
using HeuristicLab.Algorithms.DataAnalysis;
using HeuristicLab.Analysis;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
using HeuristicLab.Optimization;
using HeuristicLab.Parameters;
using HeuristicLab.PluginInfrastructure;
using HeuristicLab.Problems.DataAnalysis;
using HeuristicLab.Problems.DataAnalysis.Symbolic;
using HeuristicLab.Random;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using CancellationToken = System.Threading.CancellationToken;
using ExecutionContext = HeuristicLab.Core.ExecutionContext;

namespace HeuristicLab.Algorithms.EvolvmentModelsOfModels {
  [Item("EMMAlgorithmBase", "Base class for all EMM algorithm variants.")]
  [StorableType("A56A396B-965A-4ADE-8A2B-AE3A45F9C239")]
  public abstract class EvolvmentModelsOfModelsAlgorithmBase : FixedDataAnalysisAlgorithm<ISymbolicDataAnalysisSingleObjectiveProblem> {
    #region data members
    [Storable]
    protected IList<IEMMSolution> Solutions;
    [Storable]
    protected List<IEMMSolution> Population;
    [Storable]
    protected int EvaluatedSolutions;
    [Storable]
    protected ExecutionContext executionContext;
    [Storable]
    protected IScope globalScope;
    [Storable]
    protected ExecutionState previousExecutionState;
    [Storable]
    protected IEnumerable<ISymbolicExpressionTree> trees;

    [Storable]
    protected ExecutionState currentExecutionState;
    #endregion

    #region parameters
    private const string SeedParameterName = "Seed";
    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
    private const string PopulationSizeParameterName = "PopulationSize";
    private const string SelectorParameterName = "Selector";
    private const string GroupSizeParameterName = "GroupSize";
    private const string CrossoverProbabilityParameterName = "CrossoverProbability";
    private const string CrossoverParameterName = "Crossover";
    private const string MutationProbabilityParameterName = "MutationProbability";
    private const string MutatorParameterName = "Mutator";
    private const string MaximumEvaluatedSolutionsParameterName = "MaximumEvaluatedSolutions";
    private const string RandomParameterName = "Random";
    private const string AnalyzerParameterName = "Analyzer";
    private const string InputFileParameterName = "InputFile";
    private const string AlgorithmImplementationTypeParameterName = "AlgorithmImplementationType";
    private const string DistanceTypeParameterName = "DistanceType";
    private const string MapParameterName = "Map";
    public IValueParameter<MultiAnalyzer> AnalyzerParameter {
      get { return (ValueParameter<MultiAnalyzer>)Parameters[AnalyzerParameterName]; }
    }
    public IFixedValueParameter<IntValue> SeedParameter {
      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
    }
    public IConstrainedValueParameter<StringValue> AlgorithmImplementationTypeParameter {
      get { return (IConstrainedValueParameter<StringValue>)Parameters[AlgorithmImplementationTypeParameterName]; }
    }
    public IConstrainedValueParameter<StringValue> DistanceTypeParameter {
      get { return (IConstrainedValueParameter<StringValue>)Parameters[DistanceTypeParameterName]; }
    }
    public IConstrainedValueParameter<EMMMapBase<ISymbolicExpressionTree>> MapParameter {
      get { return (IConstrainedValueParameter<EMMMapBase<ISymbolicExpressionTree>>)Parameters[MapParameterName]; }
    }

    public IFixedValueParameter<StringValue> InputFileParameter {
      get { return (IFixedValueParameter<StringValue>)Parameters[InputFileParameterName]; }
    }
    public IFixedValueParameter<BoolValue> SetSeedRandomlyParameter {
      get { return (IFixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
    }
    private IValueParameter<IntValue> PopulationSizeParameter {
      get { return (IValueParameter<IntValue>)Parameters[PopulationSizeParameterName]; }
    }
    public IValueParameter<PercentValue> CrossoverProbabilityParameter {
      get { return (IValueParameter<PercentValue>)Parameters[CrossoverProbabilityParameterName]; }
    }
    public IValueParameter<IntValue> GroupSizeParameter {
      get { return (IValueParameter<IntValue>)Parameters[GroupSizeParameterName]; }
    }
    public IConstrainedValueParameter<ICrossover> CrossoverParameter {
      get { return (IConstrainedValueParameter<ICrossover>)Parameters[CrossoverParameterName]; }
    }
    public IConstrainedValueParameter<ISelector> SelectorParameter {
      get { return (IConstrainedValueParameter<ISelector>)Parameters[SelectorParameterName]; }
    }
    public IValueParameter<PercentValue> MutationProbabilityParameter {
      get { return (IValueParameter<PercentValue>)Parameters[MutationProbabilityParameterName]; }
    }
    public IConstrainedValueParameter<IManipulator> MutatorParameter {
      get { return (IConstrainedValueParameter<IManipulator>)Parameters[MutatorParameterName]; }
    }
    public IValueParameter<IntValue> MaximumEvaluatedSolutionsParameter {
      get { return (IValueParameter<IntValue>)Parameters[MaximumEvaluatedSolutionsParameterName]; }
    }
    public IValueParameter<IRandom> RandomParameter {
      get { return (IValueParameter<IRandom>)Parameters[RandomParameterName]; }
    }
    #endregion

    #region parameter properties
    public ValueParameter<IntValue> ElitesParameter {
      get { return (ValueParameter<IntValue>)Parameters["Elites"]; }
    }
    public int Seed {
      get { return SeedParameter.Value.Value; }
      set { SeedParameter.Value.Value = value; }
    }
    public StringValue AlgorithmImplemetationType {
      get { return AlgorithmImplementationTypeParameter.Value; }
      set { AlgorithmImplementationTypeParameter.Value.Value = value.Value; }
    }
    public StringValue DistanceType {
      get { return DistanceTypeParameter.Value; }
      set { DistanceTypeParameter.Value.Value = value.Value; }
    }
    public EMMMapBase<ISymbolicExpressionTree> Map {
      get { return MapParameter.Value; }
      set { MapParameter.Value = value; }
    }
    public StringValue InputFile {
      get { return InputFileParameter.Value; }
      set { InputFileParameter.Value.Value = value.Value; }
    }
    public bool SetSeedRandomly {
      get { return SetSeedRandomlyParameter.Value.Value; }
      set { SetSeedRandomlyParameter.Value.Value = value; }
    }
    public IntValue PopulationSize {
      get { return PopulationSizeParameter.Value; }
      set { PopulationSizeParameter.Value = value; }
    }
    public PercentValue CrossoverProbability {
      get { return CrossoverProbabilityParameter.Value; }
      set { CrossoverProbabilityParameter.Value = value; }
    }
    public IntValue GroupSize {
      get { return GroupSizeParameter.Value; }
      set { GroupSizeParameter.Value = value; }
    }
    public ICrossover Crossover {
      get { return CrossoverParameter.Value; }
      set { CrossoverParameter.Value = value; }
    }
    public ISelector Selector {
      get { return SelectorParameter.Value; }
      set { SelectorParameter.Value = value; }
    }
    public PercentValue MutationProbability {
      get { return MutationProbabilityParameter.Value; }
      set { MutationProbabilityParameter.Value = value; }
    }
    public IManipulator Mutator {
      get { return MutatorParameter.Value; }
      set { MutatorParameter.Value = value; }
    }
    public MultiAnalyzer Analyzer {
      get { return AnalyzerParameter.Value; }
      set { AnalyzerParameter.Value = value; }
    }
    public IntValue MaximumEvaluatedSolutions {
      get { return MaximumEvaluatedSolutionsParameter.Value; }
      set { MaximumEvaluatedSolutionsParameter.Value = value; }
    }
    public IntValue Elites {
      get { return ElitesParameter.Value; }
    }
    #endregion

    #region constructors
    public EvolvmentModelsOfModelsAlgorithmBase() {

      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
      Parameters.Add(new FixedValueParameter<StringValue>(InputFileParameterName, "The file with set of models that will be.", new StringValue("input.txt")));
      Parameters.Add(new ConstrainedValueParameter<StringValue>(AlgorithmImplementationTypeParameterName, "The Type of possible algorithm, implementation, choose one: OnlyMap, Full, Read."));
      Parameters.Add(new ConstrainedValueParameter<StringValue>(DistanceTypeParameterName, "The Type of possible distance calculator for case of only distance calculation."));
      Parameters.Add(new ConstrainedValueParameter<EMMMapBase<ISymbolicExpressionTree>>(MapParameterName, "The type of map creation algorithm. Use one from: IslandMap, NetworkMap."));
      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
      Parameters.Add(new ValueParameter<IntValue>(PopulationSizeParameterName, "The size of the population of Solutions.", new IntValue(100)));
      Parameters.Add(new ConstrainedValueParameter<ISelector>(SelectorParameterName, "The operator used to select parents."));
      Parameters.Add(new ValueParameter<PercentValue>(CrossoverProbabilityParameterName, "The probability that the crossover operator is applied.", new PercentValue(0.9)));
      Parameters.Add(new ValueParameter<IntValue>(GroupSizeParameterName, "The GoupSize that the Selector operator is applied.", new IntValue(3)));
      Parameters.Add(new ConstrainedValueParameter<ICrossover>(CrossoverParameterName, "The operator used to cross Solutions."));
      Parameters.Add(new ValueParameter<PercentValue>(MutationProbabilityParameterName, "The probability that the mutation operator is applied on a solution.", new PercentValue(0.25)));
      Parameters.Add(new ConstrainedValueParameter<IManipulator>(MutatorParameterName, "The operator used to mutate Solutions."));
      Parameters.Add(new ValueParameter<MultiAnalyzer>("Analyzer", "The operator used to analyze each generation.", new MultiAnalyzer()));
      Parameters.Add(new ValueParameter<IntValue>(MaximumEvaluatedSolutionsParameterName, "The maximum number of evaluated Solutions (approximately).", new IntValue(100_000)));
      Parameters.Add(new ValueParameter<IRandom>(RandomParameterName, new MersenneTwister()));
      Parameters.Add(new ValueParameter<IntValue>("Elites", "The number of elite Solutions which are kept in each generation.", new IntValue(1)));
      foreach (ISelector selector in ApplicationManager.Manager.GetInstances<ISelector>().Where(x => !(x is IMultiObjectiveSelector)).OrderBy(x => x.Name))
        SelectorParameter.ValidValues.Add(selector);
      ISelector proportionalSelector = SelectorParameter.ValidValues.FirstOrDefault(x => x.GetType().Name.Equals("ProportionalSelector"));
      if (proportionalSelector != null) SelectorParameter.Value = proportionalSelector;
      ParameterizeSelectors();

      //begin hack ...
      InputFile.ValueChanged += InputFile_ValueChanged;
      InfixExpressionParser parser = new InfixExpressionParser();
      trees = File.ReadAllLines(InputFileParameter.Value.Value).Select(parser.Parse).ToArray();
      // end hack

      ProblemChanged += EvolvmentModelsOfModelsAlgorithmBase_ProblemChanged;
      MapParameterUpdate();

    }

    // also hack vvvvvvvvv
    private void InputFile_ValueChanged(object sender, EventArgs e) {
      InfixExpressionParser parser = new InfixExpressionParser();
      trees = File.ReadAllLines(InputFileParameter.Value.Value).Select(parser.Parse);
    }
    // remove again ^^^^^^^^^ 

    private void EvolvmentModelsOfModelsAlgorithmBase_ProblemChanged(object sender, EventArgs e) {
      if (Problem != null) {
        Problem.SymbolicExpressionTreeInterpreter = new SymbolicDataAnalysisExpressionTreeBatchInterpreter();
      }
    }
    protected void MapParameterUpdate() {
      var mapTypes = new EMMMapBase<ISymbolicExpressionTree>[]
      {
        new EMMIslandMap(),
        new EMMNetworkMap(),
        new EMMDisatanceMap(),
        new EMMRankMap(),
        new EMMSucsessMap (),
        new EMMZeroMap ()
      };
      foreach (var t in mapTypes) {
        MapParameter.ValidValues.Add(t);
      }
      var algorithmType = new StringValue[]
        {
          new StringValue("Full"),
          new StringValue("Read"),
        };
      foreach (var t in algorithmType) {
        AlgorithmImplementationTypeParameter.ValidValues.Add(t);
      }
      var distanceType = new StringValue[]
        {
          new StringValue("MSE"),
          new StringValue("PearsonsRSquared"),
          new StringValue ("Covariance"),
          new StringValue ("MaxAbsoluteError"),
          new StringValue ("MeanAbsoluteError"),
          new StringValue ("Symbolic")
        };
      foreach (var t in distanceType) {
        DistanceTypeParameter.ValidValues.Add(t);
      }
    }

    protected EvolvmentModelsOfModelsAlgorithmBase(EvolvmentModelsOfModelsAlgorithmBase original, Cloner cloner) : base(original, cloner) {
      EvaluatedSolutions = original.EvaluatedSolutions;
      previousExecutionState = original.previousExecutionState;

      if (original.Solutions != null) {
        Solutions = original.Solutions.Select(cloner.Clone).ToArray();
      }

      if (original.Population != null) {
        Population = original.Population.Select(cloner.Clone).ToList();
      }

      if (original.executionContext != null) {
        executionContext = cloner.Clone(original.executionContext);
      }

      if (original.globalScope != null) {
        globalScope = cloner.Clone(original.globalScope);
      }
      // hack 
      if (original.trees != null) {
        trees = original.trees.Select(x => cloner.Clone(x)).ToArray();
      }
    }

    [StorableConstructor]
    protected EvolvmentModelsOfModelsAlgorithmBase(StorableConstructorFlag _) : base(_) { }
    #endregion


    public override void Prepare() {
      base.Prepare();
    }

    protected override void Initialize(CancellationToken cancellationToken) {
      base.Initialize(cancellationToken);
    }

    public override bool SupportsPause => true;

    // implements random number generation from https://en.wikipedia.org/wiki/Dirichlet_distribution#Random_number_generation

    public IList<IEMMSolution> GetResult(IRandom random) {
      return Population;
    }

    #region operator wiring and events
    private void ParameterizeStochasticOperator(IOperator op) {
      IStochasticOperator stochasticOp = op as IStochasticOperator;
      if (stochasticOp != null) {
        stochasticOp.RandomParameter.ActualName = "Random";
        stochasticOp.RandomParameter.Hidden = true;
      }
    }
    private void ParameterizeSelectors() {
      foreach (ISelector selector in SelectorParameter.ValidValues) {
        selector.CopySelected = new BoolValue(true);
        selector.NumberOfSelectedSubScopesParameter.Value = new IntValue(2 * (PopulationSizeParameter.Value.Value - ElitesParameter.Value.Value));
        selector.NumberOfSelectedSubScopesParameter.Hidden = true;
        ParameterizeStochasticOperator(selector);
      }
      if (Problem != null) {
        foreach (ISingleObjectiveSelector selector in SelectorParameter.ValidValues.OfType<ISingleObjectiveSelector>()) {
          selector.MaximizationParameter.ActualName = Problem.MaximizationParameter.Name;
          selector.MaximizationParameter.Hidden = true;
          selector.QualityParameter.ActualName = Problem.Evaluator.QualityParameter.ActualName;
          selector.QualityParameter.Hidden = true;
        }
      }
    }
    protected void ExecuteOperation(ExecutionContext executionContext, CancellationToken cancellationToken, IOperation operation) {
      Stack<IOperation> executionStack = new Stack<IOperation>();
      executionStack.Push(operation);
      while (executionStack.Count > 0) {
        cancellationToken.ThrowIfCancellationRequested();
        IOperation next = executionStack.Pop();
        if (next is OperationCollection) {
          OperationCollection coll = (OperationCollection)next;
          for (int i = coll.Count - 1; i >= 0; i--)
            if (coll[i] != null) executionStack.Push(coll[i]);
        } else if (next is IAtomicOperation) {
          IAtomicOperation op = (IAtomicOperation)next;
          next = op.Operator.Execute((IExecutionContext)op, cancellationToken);
          if (next != null) executionStack.Push(next);
        }
      }
    }

    private void UpdateAnalyzers() {
      Analyzer.Operators.Clear();
      if (Problem != null) {
        foreach (IAnalyzer analyzer in Problem.Operators.OfType<IAnalyzer>()) {
          foreach (IScopeTreeLookupParameter param in analyzer.Parameters.OfType<IScopeTreeLookupParameter>())
            param.Depth = 1;
          Analyzer.Operators.Add(analyzer, analyzer.EnabledByDefault);
        }
      }
    }

    private void UpdateCrossovers() {
      ICrossover oldCrossover = CrossoverParameter.Value;
      CrossoverParameter.ValidValues.Clear();
      ICrossover defaultCrossover = Problem.Operators.OfType<ICrossover>().FirstOrDefault();

      foreach (ICrossover crossover in Problem.Operators.OfType<ICrossover>().OrderBy(x => x.Name))
        CrossoverParameter.ValidValues.Add(crossover);

      if (oldCrossover != null) {
        ICrossover crossover = CrossoverParameter.ValidValues.FirstOrDefault(x => x.GetType() == oldCrossover.GetType());
        if (crossover != null) CrossoverParameter.Value = crossover;
        else oldCrossover = null;
      }
      if (oldCrossover == null && defaultCrossover != null)
        CrossoverParameter.Value = defaultCrossover;
    }

    private void UpdateMutators() {
      IManipulator oldMutator = MutatorParameter.Value;
      MutatorParameter.ValidValues.Clear();
      IManipulator defaultMutator = Problem.Operators.OfType<IManipulator>().FirstOrDefault();

      foreach (IManipulator mutator in Problem.Operators.OfType<IManipulator>().OrderBy(x => x.Name))
        MutatorParameter.ValidValues.Add(mutator);

      if (oldMutator != null) {
        IManipulator mutator = MutatorParameter.ValidValues.FirstOrDefault(x => x.GetType() == oldMutator.GetType());
        if (mutator != null) MutatorParameter.Value = mutator;
        else oldMutator = null;
      }

      if (oldMutator == null && defaultMutator != null)
        MutatorParameter.Value = defaultMutator;
    }
    private void UpdateSelectors() {
      ISelector oldSelector = SelectorParameter.Value;
      SelectorParameter.ValidValues.Clear();
      ISelector defaultSelector = Problem.Operators.OfType<ISelector>().FirstOrDefault();

      foreach (ISelector selector in Problem.Operators.OfType<ISelector>().OrderBy(x => x.Name))
        SelectorParameter.ValidValues.Add(selector);

      if (oldSelector != null) {
        ISelector selector = SelectorParameter.ValidValues.FirstOrDefault(x => x.GetType() == oldSelector.GetType());
        if (selector != null) SelectorParameter.Value = selector;
        else oldSelector = null;
      }

      if (oldSelector == null && defaultSelector != null)
        SelectorParameter.Value = defaultSelector;
    }
    protected override void OnProblemChanged() {
      UpdateCrossovers();
      UpdateMutators();
      UpdateAnalyzers();
      base.OnProblemChanged();
    }

    protected override void OnExecutionStateChanged() {
      previousExecutionState = currentExecutionState;
      currentExecutionState = ExecutionState;
      base.OnExecutionStateChanged();
    }

    protected override void OnStopped() {
      if (Solutions != null) {
        Solutions.Clear();
      }
      if (Population != null) {
        Population.Clear();
      }
      //if (offspringPopulation != null) {
      //  offspringPopulation.Clear();
      //}
      //if (jointPopulation != null) {
      //  jointPopulation.Clear();
      //}
      executionContext.Scope.SubScopes.Clear();
      base.OnStopped();
    }
    #endregion
  }
}