#region License Information
/* HeuristicLab
 * Copyright (C) 2002-2010 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */
#endregion

using System.Linq;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Operators;
using HeuristicLab.Optimization;
using HeuristicLab.Parameters;
using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
using HeuristicLab.Problems.DataAnalysis.Regression.Symbolic;
using HeuristicLab.Problems.DataAnalysis.Symbolic;
using System.Collections.Generic;
using HeuristicLab.Problems.DataAnalysis.Symbolic.Symbols;
using HeuristicLab.Problems.DataAnalysis;
using System;

using HeuristicLab.Problems.DataAnalysis.Evaluators;
using HeuristicLab.Problems.DataAnalysis.Regression;
using HeuristicLab.Analysis;
using HeuristicLab.Problems.DataAnalysis.MultiVariate.TimeSeriesPrognosis.Symbolic.Evaluators;
using HeuristicLab.Problems.DataAnalysis.MultiVariate.Regression.Symbolic;
using HeuristicLab.Problems.DataAnalysis.MultiVariate.TimeSeriesPrognosis.Symbolic.Interfaces;

namespace HeuristicLab.Problems.DataAnalysis.MultiVariate.TimeSeriesPrognosis.Symbolic.Analyzers {
  /// <summary>
  /// An operator that analyzes the validation best scaled symbolic time series prognosis solution.
  /// </summary>
  [Item("ValidationBestScaledSymbolicTimeSeriesPrognosisSolutionAnalyzer", "An operator that analyzes the validation best scaled symbolic time series prognosis solution.")]
  [StorableClass]
  public sealed class ValidationBestScaledSymbolicTimeSeriesPrognosisSolutionAnalyzer : SingleSuccessorOperator, IAnalyzer {
    private const string RandomParameterName = "Random";
    private const string SymbolicExpressionTreeParameterName = "SymbolicExpressionTree";
    private const string SymbolicExpressionTreeInterpreterParameterName = "SymbolicExpressionTreeInterpreter";
    private const string EvaluatorParameterName = "Evaluator";
    private const string MaximizationParameterName = "Maximization";
    private const string ProblemDataParameterName = "ProblemData";
    private const string ValidationSamplesStartParameterName = "SamplesStart";
    private const string ValidationSamplesEndParameterName = "SamplesEnd";
    private const string QualityParameterName = "Quality";
    private const string ScaledQualityParameterName = "ScaledQuality";
    private const string UpperEstimationLimitParameterName = "UpperEstimationLimit";
    private const string LowerEstimationLimitParameterName = "LowerEstimationLimit";
    private const string ValidationPredictionHorizonParameterName = "ValidationPredictionHorizon";
    private const string ModelPredictionHorizonParameterName = "ModelPredictionHorizon";
    private const string ConditionVariableParameterName = "ConditionVariableName";
    private const string ResultsParameterName = "Results";
    private const string VariableFrequenciesParameterName = "VariableFrequencies";
    private const string RelativeNumberOfEvaluatedSamplesParameterName = "RelativeNumberOfEvaluatedSamples";

    private const string BestSolutionParameterName = "Best solution (validation)";
    private const string BestSolutionQualityParameterName = "Best solution quality (validation)";
    private const string CurrentBestValidationQualityParameterName = "Current best validation quality";
    private const string BestSolutionQualityValuesParameterName = "Validation Quality";
    private const string BestKnownQualityParameterName = "BestKnownQuality";
    private const string GenerationsParameterName = "Generations";

    private const string BestSolutionMeanSquaredErrorTrainingParameterName = "Best validation solution mean squared error (training)";
    private const string BestSolutionMeanSquaredErrorTestParameterName = "Best validation solution mean squared error (test)";
    private const string BestSolutionRSquaredTrainingParameterName = "Best validation solution Rē (training)";
    private const string BestSolutionRSquaredTestParameterName = "Best validation solution Rē (test)";
    private const string BestSolutionDirectionalSymmetryTrainingParameterName = "Best validation solution directional symmetry (training)";
    private const string BestSolutionDirectionalSymmetryTestParameterName = "Best validation solution directional symmetry (test)";
    private const string BestSolutionTheilsUTrainingParameterName = "Best validation solution Theil's U (training)";
    private const string BestSolutionTheilsUTestParameterName = "Best validation solution Theil's U (test)";
    private const string BestSolutionTheilsUTrendTrainingParameterName = "Best validation solution Theil's U with trend (training)";
    private const string BestSolutionTheilsUTrendTestParameterName = "Best validation solution Theil's U with trend (test)";

    #region parameter properties
    public ILookupParameter<IRandom> RandomParameter {
      get { return (ILookupParameter<IRandom>)Parameters[RandomParameterName]; }
    }
    public ScopeTreeLookupParameter<SymbolicExpressionTree> SymbolicExpressionTreeParameter {
      get { return (ScopeTreeLookupParameter<SymbolicExpressionTree>)Parameters[SymbolicExpressionTreeParameterName]; }
    }
    public OptionalValueParameter<StringValue> ConditionVariableNameParameter {
      get { return (OptionalValueParameter<StringValue>)Parameters[ConditionVariableParameterName]; }
    }
    public IValueLookupParameter<ISymbolicTimeSeriesExpressionInterpreter> SymbolicExpressionTreeInterpreterParameter {
      get { return (IValueLookupParameter<ISymbolicTimeSeriesExpressionInterpreter>)Parameters[SymbolicExpressionTreeInterpreterParameterName]; }
    }
    public IValueLookupParameter<MultiVariateDataAnalysisProblemData> ProblemDataParameter {
      get { return (IValueLookupParameter<MultiVariateDataAnalysisProblemData>)Parameters[ProblemDataParameterName]; }
    }
    public ILookupParameter<ISingleObjectiveSymbolicTimeSeriesPrognosisEvaluator> EvaluatorParameter {
      get { return (ILookupParameter<ISingleObjectiveSymbolicTimeSeriesPrognosisEvaluator>)Parameters[EvaluatorParameterName]; }
    }
    public IValueLookupParameter<IntValue> ValidationSamplesStartParameter {
      get { return (IValueLookupParameter<IntValue>)Parameters[ValidationSamplesStartParameterName]; }
    }
    public IValueLookupParameter<IntValue> ValidationSamplesEndParameter {
      get { return (IValueLookupParameter<IntValue>)Parameters[ValidationSamplesEndParameterName]; }
    }
    public IValueLookupParameter<DoubleArray> UpperEstimationLimitParameter {
      get { return (IValueLookupParameter<DoubleArray>)Parameters[UpperEstimationLimitParameterName]; }
    }
    public IValueLookupParameter<DoubleArray> LowerEstimationLimitParameter {
      get { return (IValueLookupParameter<DoubleArray>)Parameters[LowerEstimationLimitParameterName]; }
    }
    public IValueLookupParameter<IntValue> ValidationPredictionHorizonParameter {
      get { return (IValueLookupParameter<IntValue>)Parameters[ValidationPredictionHorizonParameterName]; }
    }
    public IValueLookupParameter<IntValue> ModelPredictionHorizonParameter {
      get { return (IValueLookupParameter<IntValue>)Parameters[ModelPredictionHorizonParameterName]; }
    }
    public ILookupParameter<SymbolicTimeSeriesPrognosisSolution> BestSolutionParameter {
      get { return (ILookupParameter<SymbolicTimeSeriesPrognosisSolution>)Parameters[BestSolutionParameterName]; }
    }
    public ILookupParameter<IntValue> GenerationsParameter {
      get { return (ILookupParameter<IntValue>)Parameters[GenerationsParameterName]; }
    }
    public ILookupParameter<DoubleValue> BestSolutionQualityParameter {
      get { return (ILookupParameter<DoubleValue>)Parameters[BestSolutionQualityParameterName]; }
    }
    public ILookupParameter<ResultCollection> ResultsParameter {
      get { return (ILookupParameter<ResultCollection>)Parameters[ResultsParameterName]; }
    }
    public ILookupParameter<DoubleValue> BestKnownQualityParameter {
      get { return (ILookupParameter<DoubleValue>)Parameters[BestKnownQualityParameterName]; }
    }
    public ILookupParameter<DataTable> VariableFrequenciesParameter {
      get { return (ILookupParameter<DataTable>)Parameters[VariableFrequenciesParameterName]; }
    }
    public IValueParameter<PercentValue> RelativeNumberOfEvaluatedSamplesParameter {
      get { return (IValueParameter<PercentValue>)Parameters[RelativeNumberOfEvaluatedSamplesParameterName]; }
    }
    public ILookupParameter<BoolValue> MaximizationParameter {
      get { return (ILookupParameter<BoolValue>)Parameters[MaximizationParameterName]; }
    }
    public ILookupParameter<DoubleArray> BestSolutionMeanSquaredErrorTrainingParameter {
      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionMeanSquaredErrorTrainingParameterName]; }
    }
    public ILookupParameter<DoubleArray> BestSolutionMeanSquaredErrorTestParameter {
      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionMeanSquaredErrorTestParameterName]; }
    }
    public ILookupParameter<DoubleArray> BestSolutionRSquaredTrainingParameter {
      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionRSquaredTrainingParameterName]; }
    }
    public ILookupParameter<DoubleArray> BestSolutionRSquaredTestParameter {
      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionRSquaredTestParameterName]; }
    }
    public ILookupParameter<DoubleArray> BestSolutionDirectionalSymmetryTrainingParameter {
      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionDirectionalSymmetryTrainingParameterName]; }
    }
    public ILookupParameter<DoubleArray> BestSolutionDirectionalSymmetryTestParameter {
      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionDirectionalSymmetryTestParameterName]; }
    }
    public ILookupParameter<DoubleArray> BestSolutionTheilsUTrainingParameter {
      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionTheilsUTrainingParameterName]; }
    }
    public ILookupParameter<DoubleArray> BestSolutionTheilsUTestParameter {
      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionTheilsUTestParameterName]; }
    }
    public ILookupParameter<DoubleArray> BestSolutionTheilsUTrendTrainingParameter {
      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionTheilsUTrendTrainingParameterName]; }
    }
    public ILookupParameter<DoubleArray> BestSolutionTheilsUTrendTestParameter {
      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionTheilsUTrendTestParameterName]; }
    }
    #endregion
    #region properties
    public IRandom Random {
      get { return RandomParameter.ActualValue; }
    }
    public ItemArray<SymbolicExpressionTree> SymbolicExpressionTree {
      get { return SymbolicExpressionTreeParameter.ActualValue; }
    }
    public ISymbolicTimeSeriesExpressionInterpreter SymbolicExpressionTreeInterpreter {
      get { return SymbolicExpressionTreeInterpreterParameter.ActualValue; }
    }
    public MultiVariateDataAnalysisProblemData ProblemData {
      get { return ProblemDataParameter.ActualValue; }
    }
    public IntValue ValidiationSamplesStart {
      get { return ValidationSamplesStartParameter.ActualValue; }
    }
    public IntValue ValidationSamplesEnd {
      get { return ValidationSamplesEndParameter.ActualValue; }
    }
    public DoubleArray UpperEstimationLimit {
      get { return UpperEstimationLimitParameter.ActualValue; }
    }
    public DoubleArray LowerEstimationLimit {
      get { return LowerEstimationLimitParameter.ActualValue; }
    }
    public IntValue ValidationPredictionHorizon {
      get { return ValidationPredictionHorizonParameter.ActualValue; }
    }
    public IntValue ModelPredictionHorizon {
      get { return ModelPredictionHorizonParameter.ActualValue; }
    }
    public StringValue ConditionVariableName {
      get { return ConditionVariableNameParameter.Value; }
    }
    public ResultCollection Results {
      get { return ResultsParameter.ActualValue; }
    }
    public DataTable VariableFrequencies {
      get { return VariableFrequenciesParameter.ActualValue; }
    }
    public IntValue Generations {
      get { return GenerationsParameter.ActualValue; }
    }
    public PercentValue RelativeNumberOfEvaluatedSamples {
      get { return RelativeNumberOfEvaluatedSamplesParameter.Value; }
    }
    public BoolValue Maximization {
      get { return MaximizationParameter.ActualValue; }
    }
    public DoubleArray BestSolutionMeanSquaredErrorTraining {
      get { return BestSolutionMeanSquaredErrorTrainingParameter.ActualValue; }
      set { BestSolutionMeanSquaredErrorTrainingParameter.ActualValue = value; }
    }
    public DoubleArray BestSolutionMeanSquaredErrorTest {
      get { return BestSolutionMeanSquaredErrorTestParameter.ActualValue; }
      set { BestSolutionMeanSquaredErrorTestParameter.ActualValue = value; }
    }
    public DoubleArray BestSolutionRSquaredTraining {
      get { return BestSolutionRSquaredTrainingParameter.ActualValue; }
      set { BestSolutionRSquaredTrainingParameter.ActualValue = value; }
    }
    public DoubleArray BestSolutionRSquaredTest {
      get { return BestSolutionRSquaredTestParameter.ActualValue; }
      set { BestSolutionRSquaredTestParameter.ActualValue = value; }
    }
    public DoubleArray BestSolutionDirectionalSymmetryTraining {
      get { return BestSolutionDirectionalSymmetryTrainingParameter.ActualValue; }
      set { BestSolutionDirectionalSymmetryTrainingParameter.ActualValue = value; }
    }
    public DoubleArray BestSolutionDirectionalSymmetryTest {
      get { return BestSolutionDirectionalSymmetryTestParameter.ActualValue; }
      set { BestSolutionDirectionalSymmetryTestParameter.ActualValue = value; }
    }
    public DoubleArray BestSolutionTheilsUTraining {
      get { return BestSolutionTheilsUTrainingParameter.ActualValue; }
      set { BestSolutionTheilsUTrainingParameter.ActualValue = value; }
    }
    public DoubleArray BestSolutionTheilsUTest {
      get { return BestSolutionTheilsUTestParameter.ActualValue; }
      set { BestSolutionTheilsUTestParameter.ActualValue = value; }
    }
    public DoubleArray BestSolutionTheilsUTrendTraining {
      get { return BestSolutionTheilsUTrendTrainingParameter.ActualValue; }
      set { BestSolutionTheilsUTrendTrainingParameter.ActualValue = value; }
    }
    public DoubleArray BestSolutionTheilsUTrendTest {
      get { return BestSolutionTheilsUTrendTestParameter.ActualValue; }
      set { BestSolutionTheilsUTrendTestParameter.ActualValue = value; }
    }
    #endregion

    [StorableConstructor]
    protected ValidationBestScaledSymbolicTimeSeriesPrognosisSolutionAnalyzer(bool deserializing) : base(deserializing) { }
    protected ValidationBestScaledSymbolicTimeSeriesPrognosisSolutionAnalyzer(ValidationBestScaledSymbolicTimeSeriesPrognosisSolutionAnalyzer original, Cloner cloner)
      : base(original, cloner) {
    }
    public ValidationBestScaledSymbolicTimeSeriesPrognosisSolutionAnalyzer()
      : base() {
      Parameters.Add(new LookupParameter<IRandom>(RandomParameterName, "A random number generator."));
      Parameters.Add(new ScopeTreeLookupParameter<SymbolicExpressionTree>(SymbolicExpressionTreeParameterName, "The symbolic expression trees to analyze."));
      Parameters.Add(new OptionalValueParameter<StringValue>(ConditionVariableParameterName, "The name of the condition variable indicating if a row should be considered for evaluation or not."));
      Parameters.Add(new ValueLookupParameter<ISymbolicTimeSeriesExpressionInterpreter>(SymbolicExpressionTreeInterpreterParameterName, "The interpreter that should be used for the analysis of symbolic expression trees."));
      Parameters.Add(new ValueLookupParameter<MultiVariateDataAnalysisProblemData>(ProblemDataParameterName, "The problem data for which the symbolic expression tree is a solution."));
      Parameters.Add(new ValueLookupParameter<IntValue>(ValidationSamplesStartParameterName, "The first index of the validation partition of the data set."));
      Parameters.Add(new ValueLookupParameter<IntValue>(ValidationSamplesEndParameterName, "The last index of the validation partition of the data set."));
      Parameters.Add(new ValueLookupParameter<IntValue>(ValidationPredictionHorizonParameterName, "The number of time steps for which to create a forecast for the validation procedure."));
      Parameters.Add(new ValueLookupParameter<IntValue>(ModelPredictionHorizonParameterName, "Prediction horizont stored in the validation best model."));
      Parameters.Add(new LookupParameter<ISingleObjectiveSymbolicTimeSeriesPrognosisEvaluator>(EvaluatorParameterName, ""));
      Parameters.Add(new ValueLookupParameter<DoubleArray>(UpperEstimationLimitParameterName, "The upper estimation limit that was set for the evaluation of the symbolic expression trees."));
      Parameters.Add(new ValueLookupParameter<DoubleArray>(LowerEstimationLimitParameterName, "The lower estimation limit that was set for the evaluation of the symbolic expression trees."));
      Parameters.Add(new LookupParameter<SymbolicTimeSeriesPrognosisSolution>(BestSolutionParameterName, "The best symbolic time series prognosis solution."));
      Parameters.Add(new LookupParameter<IntValue>(GenerationsParameterName, "The number of generations calculated so far."));
      Parameters.Add(new LookupParameter<DoubleValue>(BestSolutionQualityParameterName, "The quality of the best symbolic regression solution."));
      Parameters.Add(new LookupParameter<ResultCollection>(ResultsParameterName, "The result collection where the best symbolic regression solution should be stored."));
      Parameters.Add(new LookupParameter<DoubleValue>(BestKnownQualityParameterName, "The best known (validation) quality achieved on the data set."));
      Parameters.Add(new LookupParameter<DataTable>(VariableFrequenciesParameterName, "The variable frequencies table to use for the calculation of variable impacts"));
      Parameters.Add(new ValueParameter<PercentValue>(RelativeNumberOfEvaluatedSamplesParameterName, "The relative number of samples of the dataset partition, which should be randomly chosen for evaluation between the start and end index.", new PercentValue(1)));
      Parameters.Add(new LookupParameter<BoolValue>(MaximizationParameterName));
      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionMeanSquaredErrorTrainingParameterName));
      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionMeanSquaredErrorTestParameterName));
      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionRSquaredTrainingParameterName));
      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionRSquaredTestParameterName));
      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionDirectionalSymmetryTrainingParameterName));
      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionDirectionalSymmetryTestParameterName));
      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionTheilsUTrainingParameterName));
      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionTheilsUTestParameterName));
      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionTheilsUTrendTrainingParameterName));
      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionTheilsUTrendTestParameterName));
    }

    public override IDeepCloneable Clone(Cloner cloner) {
      return new ValidationBestScaledSymbolicTimeSeriesPrognosisSolutionAnalyzer(this, cloner);
    }
    [StorableHook(Persistence.Default.CompositeSerializers.Storable.HookType.AfterDeserialization)]
    private void AfterDeserialization() {

    }

    public override IOperation Apply() {
      var trees = SymbolicExpressionTree;

      ISingleObjectiveSymbolicTimeSeriesPrognosisEvaluator evaluator = EvaluatorParameter.ActualValue;

      int trainingStart = ProblemData.TrainingSamplesStart.Value;
      int trainingEnd = ProblemData.TrainingSamplesEnd.Value;
      int testStart = ProblemData.TestSamplesStart.Value;
      int testEnd = ProblemData.TestSamplesEnd.Value;

      #region validation best model
      int validationStart = ValidiationSamplesStart.Value;
      int validationEnd = ValidationSamplesEnd.Value;
      int rowCount = (int)Math.Ceiling((validationEnd - validationStart) * RelativeNumberOfEvaluatedSamples.Value);
      IEnumerable<int> rows = RandomEnumerable.SampleRandomNumbers(Random.Next(), validationStart, validationEnd, rowCount);
      double bestValidationQuality = Maximization.Value ? double.MinValue : double.MaxValue;
      SymbolicExpressionTree bestTree = null;
      string conditionalVariableName = ConditionVariableName == null ? null : ConditionVariableName.Value;
      if (conditionalVariableName != null) {
        rows = from row in rows
               where !ProblemData.Dataset[conditionalVariableName, row].IsAlmost(0.0)
               select row;
      }

      foreach (var tree in trees) {
        double validationQuality;
        validationQuality = evaluator.Evaluate(tree, ProblemData,
          SymbolicExpressionTreeInterpreter, rows, ValidationPredictionHorizon.Value, LowerEstimationLimit, UpperEstimationLimit);
        if ((Maximization.Value && validationQuality > bestValidationQuality) ||
           (!Maximization.Value && validationQuality < bestValidationQuality)) {
          bestValidationQuality = validationQuality;
          bestTree = tree;
        }
      }


      if (BestSolutionQualityParameter.ActualValue == null ||
        (Maximization.Value && BestSolutionQualityParameter.ActualValue.Value < bestValidationQuality) ||
        (!Maximization.Value && BestSolutionQualityParameter.ActualValue.Value > bestValidationQuality)) {
        var scaledTree = GetScaledTree(bestTree);
        var model = new SymbolicTimeSeriesPrognosisModel((ISymbolicTimeSeriesExpressionInterpreter)SymbolicExpressionTreeInterpreter.Clone(), scaledTree);
        model.Name = "Time series prognosis model";
        model.Description = "Best solution on validation partition found over the whole run.";

        var solution = new SymbolicTimeSeriesPrognosisSolution((MultiVariateDataAnalysisProblemData)ProblemData.Clone(), model, ModelPredictionHorizon.Value, conditionalVariableName, LowerEstimationLimit.ToArray(), UpperEstimationLimit.ToArray());
        solution.Name = BestSolutionParameterName;
        solution.Description = "Best solution on validation partition found over the whole run.";

        BestSolutionParameter.ActualValue = solution;
        BestSolutionQualityParameter.ActualValue = new DoubleValue(bestValidationQuality);

        #region calculate accuracy
        List<string> targetVariables = ProblemData.TargetVariables.CheckedItems.Select(x => x.Value.Value).ToList();

        // create a list of time series evaluators for each target variable
        Dictionary<string, List<IOnlineEvaluator>> trainingEvaluators =
          new Dictionary<string, List<IOnlineEvaluator>>();
        Dictionary<string, List<IOnlineEvaluator>> testEvaluators =
          new Dictionary<string, List<IOnlineEvaluator>>();
        foreach (string targetVariable in targetVariables) {
          trainingEvaluators.Add(targetVariable, new List<IOnlineEvaluator>());
          trainingEvaluators[targetVariable].Add(new OnlineMeanSquaredErrorEvaluator());
          trainingEvaluators[targetVariable].Add(new OnlinePearsonsRSquaredEvaluator());
          trainingEvaluators[targetVariable].Add(new OnlineDirectionalSymmetryEvaluator());
          trainingEvaluators[targetVariable].Add(new OnlineTheilsUStatisticEvaluator());
          trainingEvaluators[targetVariable].Add(new OnlineTheilsUStatisticEvaluator(10));

          testEvaluators.Add(targetVariable, new List<IOnlineEvaluator>());
          testEvaluators[targetVariable].Add(new OnlineMeanSquaredErrorEvaluator());
          testEvaluators[targetVariable].Add(new OnlinePearsonsRSquaredEvaluator());
          testEvaluators[targetVariable].Add(new OnlineDirectionalSymmetryEvaluator());
          testEvaluators[targetVariable].Add(new OnlineTheilsUStatisticEvaluator());
          testEvaluators[targetVariable].Add(new OnlineTheilsUStatisticEvaluator(10));
        }

        Evaluate(solution, solution.ProblemData.Dataset, targetVariables, conditionalVariableName, trainingStart, trainingEnd, trainingEvaluators);
        Evaluate(solution, solution.ProblemData.Dataset, targetVariables, conditionalVariableName, testStart, testEnd, testEvaluators);
        #endregion
        BestSolutionMeanSquaredErrorTraining = new DoubleArray((from variable in targetVariables
                                                                let eval = trainingEvaluators[variable].OfType<OnlineMeanSquaredErrorEvaluator>().Single()
                                                                select TryGetValue(eval))
                                                                .ToArray());
        BestSolutionMeanSquaredErrorTest = new DoubleArray((from variable in targetVariables
                                                            select TryGetValue(testEvaluators[variable].OfType<OnlineMeanSquaredErrorEvaluator>().Single()))
                                                          .ToArray());
        BestSolutionRSquaredTraining = new DoubleArray((from variable in targetVariables
                                                        select TryGetValue(trainingEvaluators[variable].OfType<OnlinePearsonsRSquaredEvaluator>().Single()))
                                                                .ToArray());
        BestSolutionRSquaredTest = new DoubleArray((from variable in targetVariables
                                                    select TryGetValue(testEvaluators[variable].OfType<OnlinePearsonsRSquaredEvaluator>().Single()))
                                                          .ToArray());
        BestSolutionDirectionalSymmetryTraining = new DoubleArray((from variable in targetVariables
                                                                   select TryGetValue(trainingEvaluators[variable].OfType<OnlineDirectionalSymmetryEvaluator>().Single()))
                                                                .ToArray());
        BestSolutionDirectionalSymmetryTest = new DoubleArray((from variable in targetVariables
                                                               select TryGetValue(testEvaluators[variable].OfType<OnlineDirectionalSymmetryEvaluator>().Single()))
                                                          .ToArray());
        BestSolutionTheilsUTraining = new DoubleArray((from variable in targetVariables
                                                       select TryGetValue(trainingEvaluators[variable].OfType<OnlineTheilsUStatisticEvaluator>().First()))
                                                                .ToArray());
        BestSolutionTheilsUTest = new DoubleArray((from variable in targetVariables
                                                   select TryGetValue(testEvaluators[variable].OfType<OnlineTheilsUStatisticEvaluator>().First()))
                                                          .ToArray());
        BestSolutionTheilsUTrendTraining = new DoubleArray((from variable in targetVariables
                                                            select TryGetValue(trainingEvaluators[variable].OfType<OnlineTheilsUStatisticEvaluator>().Skip(1).First()))
                                                                .ToArray());
        BestSolutionTheilsUTrendTest = new DoubleArray((from variable in targetVariables
                                                        select TryGetValue(testEvaluators[variable].OfType<OnlineTheilsUStatisticEvaluator>().Skip(1).First()))
                                                          .ToArray());
        if (!Results.ContainsKey(BestSolutionParameterName)) {
          for (int i = 0; i < targetVariables.Count; i++) {
            Results.Add(new Result(BestSolutionMeanSquaredErrorTrainingParameterName + " (" + targetVariables[i] + ")",
              new DoubleValue(BestSolutionMeanSquaredErrorTraining[i])));
            Results.Add(new Result(BestSolutionMeanSquaredErrorTestParameterName + " (" + targetVariables[i] + ")",
              new DoubleValue(BestSolutionMeanSquaredErrorTest[i])));
            Results.Add(new Result(BestSolutionRSquaredTrainingParameterName + " (" + targetVariables[i] + ")",
              new DoubleValue(BestSolutionRSquaredTraining[i])));
            Results.Add(new Result(BestSolutionRSquaredTestParameterName + " (" + targetVariables[i] + ")",
              new DoubleValue(BestSolutionRSquaredTest[i])));
            Results.Add(new Result(BestSolutionDirectionalSymmetryTrainingParameterName + " (" + targetVariables[i] + ")",
              new DoubleValue(BestSolutionDirectionalSymmetryTraining[i])));
            Results.Add(new Result(BestSolutionDirectionalSymmetryTestParameterName + " (" + targetVariables[i] + ")",
              new DoubleValue(BestSolutionDirectionalSymmetryTest[i])));
            Results.Add(new Result(BestSolutionTheilsUTrainingParameterName + " (" + targetVariables[i] + ")",
              new DoubleValue(BestSolutionTheilsUTraining[i])));
            Results.Add(new Result(BestSolutionTheilsUTestParameterName + " (" + targetVariables[i] + ")",
              new DoubleValue(BestSolutionTheilsUTest[i])));
            Results.Add(new Result(BestSolutionTheilsUTrendTrainingParameterName + " (" + targetVariables[i] + ")",
              new DoubleValue(BestSolutionTheilsUTrendTraining[i])));
            Results.Add(new Result(BestSolutionTheilsUTrendTestParameterName + " (" + targetVariables[i] + ")",
              new DoubleValue(BestSolutionTheilsUTrendTest[i])));
          }
        } else {
          for (int i = 0; i < targetVariables.Count; i++) {
            Results[BestSolutionMeanSquaredErrorTrainingParameterName + " (" + targetVariables[i] + ")"].Value =
              new DoubleValue(BestSolutionMeanSquaredErrorTraining[i]);
            Results[BestSolutionMeanSquaredErrorTestParameterName + " (" + targetVariables[i] + ")"].Value =
              new DoubleValue(BestSolutionMeanSquaredErrorTest[i]);
            Results[BestSolutionRSquaredTrainingParameterName + " (" + targetVariables[i] + ")"].Value =
              new DoubleValue(BestSolutionRSquaredTraining[i]);
            Results[BestSolutionRSquaredTestParameterName + " (" + targetVariables[i] + ")"].Value =
              new DoubleValue(BestSolutionRSquaredTest[i]);
            Results[BestSolutionDirectionalSymmetryTrainingParameterName + " (" + targetVariables[i] + ")"].Value =
              new DoubleValue(BestSolutionDirectionalSymmetryTraining[i]);
            Results[BestSolutionDirectionalSymmetryTestParameterName + " (" + targetVariables[i] + ")"].Value =
              new DoubleValue(BestSolutionDirectionalSymmetryTest[i]);
            Results[BestSolutionTheilsUTrainingParameterName + " (" + targetVariables[i] + ")"].Value =
              new DoubleValue(BestSolutionTheilsUTraining[i]);
            Results[BestSolutionTheilsUTestParameterName + " (" + targetVariables[i] + ")"].Value =
              new DoubleValue(BestSolutionTheilsUTest[i]);
            Results[BestSolutionTheilsUTrendTrainingParameterName + " (" + targetVariables[i] + ")"].Value =
              new DoubleValue(BestSolutionTheilsUTrendTraining[i]);
            Results[BestSolutionTheilsUTrendTestParameterName + " (" + targetVariables[i] + ")"].Value =
              new DoubleValue(BestSolutionTheilsUTrendTest[i]);
          }
        }
      }

      if (!Results.ContainsKey(BestSolutionQualityValuesParameterName)) {
        Results.Add(new Result(BestSolutionParameterName, BestSolutionParameter.ActualValue));
        Results.Add(new Result(BestSolutionQualityValuesParameterName, new DataTable(BestSolutionQualityValuesParameterName, BestSolutionQualityValuesParameterName)));
        Results.Add(new Result(BestSolutionQualityParameterName, new DoubleValue()));
        Results.Add(new Result(CurrentBestValidationQualityParameterName, new DoubleValue()));
      }
      Results[BestSolutionParameterName].Value = BestSolutionParameter.ActualValue;
      Results[BestSolutionQualityParameterName].Value = new DoubleValue(BestSolutionQualityParameter.ActualValue.Value);
      Results[CurrentBestValidationQualityParameterName].Value = new DoubleValue(bestValidationQuality);

      DataTable validationValues = (DataTable)Results[BestSolutionQualityValuesParameterName].Value;
      AddValue(validationValues, BestSolutionQualityParameter.ActualValue.Value, BestSolutionQualityParameterName, BestSolutionQualityParameterName);
      AddValue(validationValues, bestValidationQuality, CurrentBestValidationQualityParameterName, CurrentBestValidationQualityParameterName);
      #endregion
      return base.Apply();
    }

    private double TryGetValue(IOnlineEvaluator eval) {
      try {
        return eval.Value;
      }
      catch {
        return double.NaN;
      }
    }

    private SymbolicExpressionTree GetScaledTree(SymbolicExpressionTree tree) {
      double[] alpha, beta;
      int trainingStart = ProblemData.TrainingSamplesStart.Value;
      int trainingEnd = ProblemData.TrainingSamplesEnd.Value;
      IEnumerable<int> trainingRows = Enumerable.Range(trainingStart, trainingEnd - trainingStart);
      string conditionalVariableName = ConditionVariableName == null ? null : ConditionVariableName.Value;
      if (conditionalVariableName != null) {
        trainingRows = from row in trainingRows
                       where !ProblemData.Dataset[conditionalVariableName, row].IsAlmost(0.0)
                       select row;
      }

      // calculate scaling parameters based on one-step-predictions
      IEnumerable<string> selectedTargetVariables = from item in ProblemData.TargetVariables
                                                    where ProblemData.TargetVariables.ItemChecked(item)
                                                    select item.Value;
      int dimension = selectedTargetVariables.Count();

      IEnumerable<IEnumerable<double>> oneStepPredictions =
        SymbolicExpressionTreeInterpreter.GetSymbolicExpressionTreeValues(tree, ProblemData.Dataset, selectedTargetVariables, trainingRows, 1)
        .Cast<IEnumerable<double>>();
      IEnumerable<IEnumerable<double>> originalValues = from row in trainingRows
                                                        select (from targetVariable in selectedTargetVariables
                                                                select ProblemData.Dataset[targetVariable, row]);
      alpha = new double[dimension];
      beta = new double[dimension];

      SymbolicTimeSeriesPrognosisScaledNormalizedMseEvaluator.CalculateScalingParameters(originalValues, oneStepPredictions, ref beta, ref alpha);

      // scale tree for solution
      return SymbolicVectorRegressionSolutionLinearScaler.Scale(tree, beta, alpha);
    }

    private void Evaluate(SymbolicTimeSeriesPrognosisSolution solution, Dataset dataset, IEnumerable<string> targetVariables, string conditionalEvaluationVariable, int start, int end, Dictionary<string, List<IOnlineEvaluator>> evaluators) {

      for (int row = start; row < end; row++) {
        if (string.IsNullOrEmpty(conditionalEvaluationVariable) || dataset[conditionalEvaluationVariable, row] != 0) {
          // prepare evaluators for each target variable for a new prediction window
          foreach (var entry in evaluators) {
            double referenceOriginalValue = dataset[entry.Key, row - 1];
            foreach (IOnlineTimeSeriesPrognosisEvaluator evaluator in entry.Value.OfType<IOnlineTimeSeriesPrognosisEvaluator>()) {
              evaluator.StartNewPredictionWindow(referenceOriginalValue);
            }
          }


          if (string.IsNullOrEmpty(conditionalEvaluationVariable) ||
            dataset[conditionalEvaluationVariable, row] > 0) {
            int timestep = 0;
            foreach (double[] x in solution.GetPrognosis(row)) {
              int targetIndex = 0;
              if (row + timestep < dataset.Rows) {
                foreach (var targetVariable in targetVariables) {
                  double originalValue = dataset[targetVariable, row + timestep];
                  double estimatedValue = x[targetIndex];
                  if (IsValidValue(originalValue) && IsValidValue(estimatedValue)) {
                    foreach (IOnlineEvaluator evaluator in evaluators[targetVariable]) {
                      evaluator.Add(originalValue, estimatedValue);
                    }
                  }
                  targetIndex++;
                }
              }
              timestep++;
            }
          }
        }
      }
    }
    private bool IsValidValue(double d) {
      return !(double.IsNaN(d) || double.IsInfinity(d));
    }

    private static void AddValue(DataTable table, double data, string name, string description) {
      DataRow row;
      table.Rows.TryGetValue(name, out row);
      if (row == null) {
        row = new DataRow(name, description);
        row.Values.Add(data);
        table.Rows.Add(row);
      } else {
        row.Values.Add(data);
      }
    }
  }
}