source: branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/ParameterConfigurationEvaluator.cs @ 5212

using System;
using System.Linq;
using System.Threading;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Operators;
using HeuristicLab.Optimization;
using HeuristicLab.Parameters;
using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
using System.Collections.Generic;
using HeuristicLab.Algorithms.GeneticAlgorithm;
using System.Threading.Tasks;
using System.Diagnostics;
using System.Reflection;

namespace HeuristicLab.Problems.MetaOptimization {
  /// <summary>
  /// A base class for operators which evaluate TSP solutions.
  /// </summary>
  [Item("ParameterConfigurationEvaluator", "A base class for operators which evaluate Meta Optimization solutions.")]
  [StorableClass]
  public class ParameterConfigurationEvaluator : SingleSuccessorOperator, IParameterConfigurationEvaluator {
    private const double PenaltyQuality = 100000.0; // todo: use something better

    public ILookupParameter<DoubleValue> QualityParameter {
      get { return (ILookupParameter<DoubleValue>)Parameters["Quality"]; }
    }
    public ILookupParameter<EngineAlgorithm> AlgorithmParameter {
      get { return (ILookupParameter<EngineAlgorithm>)Parameters[MetaOptimizationProblem.AlgorithmTypeParameterName]; }
    }
    public ILookupParameter<IItemList<ISingleObjectiveProblem>> ProblemsParameter {
      get { return (ILookupParameter<IItemList<ISingleObjectiveProblem>>)Parameters[MetaOptimizationProblem.ProblemsParameterName]; }
    }
    public ILookupParameter<ParameterConfigurationTree> ParameterConfigurationParameter {
      get { return (ILookupParameter<ParameterConfigurationTree>)Parameters["ParameterConfigurationTree"]; }
    }
    public LookupParameter<IntValue> RepetitionsParameter {
      get { return (LookupParameter<IntValue>)Parameters[MetaOptimizationProblem.RepetitionsParameterName]; }
    }

    public LookupParameter<DoubleArray> ProblemQualityMediansParameter {
      get { return (LookupParameter<DoubleArray>)Parameters["ProblemQualityMedians"]; }
    }

    public IntValue Repetitions {
      get { return RepetitionsParameter.ActualValue; }
    }

    public ParameterConfigurationEvaluator()
      : base() {
      Parameters.Add(new LookupParameter<DoubleValue>("Quality", "The evaluated quality of the ParameterVector."));
      Parameters.Add(new LookupParameter<EngineAlgorithm>(MetaOptimizationProblem.AlgorithmTypeParameterName, "Missing description."));
      Parameters.Add(new LookupParameter<IItemList<ISingleObjectiveProblem>>(MetaOptimizationProblem.ProblemsParameterName, "Missing description."));
      Parameters.Add(new LookupParameter<ParameterConfigurationTree>("ParameterConfigurationTree", "Missing description."));
      Parameters.Add(new LookupParameter<IntValue>(MetaOptimizationProblem.RepetitionsParameterName, "Number of evaluations on one problem."));
      Parameters.Add(new LookupParameter<DoubleArray>("ProblemQualityMedians", ""));
    }

    [StorableConstructor]
    protected ParameterConfigurationEvaluator(bool deserializing) : base(deserializing) { }
    protected ParameterConfigurationEvaluator(ParameterConfigurationEvaluator original, Cloner cloner)
      : base(original, cloner) {
    }
    public override IDeepCloneable Clone(Cloner cloner) {
      return new ParameterConfigurationEvaluator(this, cloner);
    }

    public override IOperation Apply() {
      EngineAlgorithm algorithm = (EngineAlgorithm)AlgorithmParameter.ActualValue.Clone();
      IItemList<ISingleObjectiveProblem> problems = ProblemsParameter.ActualValue;

      double[] medians;
      if (ProblemQualityMediansParameter.ActualValue == null) {
        // this is generation zero. no reference qualities for scaling have been calculated yet. calculate a random individuum to get reference values
        medians = new double[problems.Count]; // todo
        for (int i = 0; i < medians.Length; i++) {
          medians[i] = 10;
        }
      } else {
        medians = ProblemQualityMediansParameter.ActualValue.ToArray();
      }

      // set parameters
      ParameterConfigurationParameter.ActualValue.Parameterize(algorithm);
      algorithm.StoreAlgorithmInEachRun = false;

      List<List<double>> qualities = new List<List<double>>();
      List<List<TimeSpan>> executionTimes = new List<List<TimeSpan>>();
      algorithm.Engine = new SequentialEngine.SequentialEngine();
      algorithm.Prepare(true);

      foreach (ISingleObjectiveProblem problem in problems) {
        algorithm.Problem = (IProblem)problem.Clone();
        var problemQualities = new List<double>();
        var problemExecutionTimes = new List<TimeSpan>();

        for (int i = 0; i < Repetitions.Value; i++) {
          algorithm.Prepare();

          AlgorithmExecutor executor = new AlgorithmExecutor(algorithm);
          executor.StartSync();

          if (algorithm.ExecutionState == ExecutionState.Paused) {
            // this parametercombination was bad. set penalty for this solution
            problemQualities.Add(PenaltyQuality); // todo: respect Maximization; problem: this messes up average value; solution: use currently worst quality*2
            problemExecutionTimes.Add(algorithm.ExecutionTime);
          } else {
            problemQualities.Add(((DoubleValue)algorithm.Results["BestQuality"].Value).Value);
            problemExecutionTimes.Add(algorithm.ExecutionTime);

            // parameters will be stored in ParameterConfigurationTree anyway. they would be redundant in runs
            algorithm.Runs.Last().Parameters.Clear();
            // but keep the problem, since this differs in runs
            algorithm.Runs.Last().Parameters.Add("Problem", problem);
          }
        }
        qualities.Add(problemQualities);
        executionTimes.Add(problemExecutionTimes);
      }
      algorithm.Prepare();

      ParameterConfigurationParameter.ActualValue.AverageExecutionTimes = new ItemList<TimeSpanValue>(executionTimes.Select(t => new TimeSpanValue(TimeSpan.FromMilliseconds(t.Average(ts => ts.TotalMilliseconds)))));
      ParameterConfigurationParameter.ActualValue.Repetitions = (IntValue)Repetitions.Clone();
      ParameterConfigurationParameter.ActualValue.BestQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray()); // todo: respect Maximization:true/false
      ParameterConfigurationParameter.ActualValue.AverageQualities = new DoubleArray(qualities.Select(q => q.Average()).ToArray());
      ParameterConfigurationParameter.ActualValue.WorstQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray()); // todo: respect Maximization:true/false
      ParameterConfigurationParameter.ActualValue.QualityVariances = new DoubleArray(qualities.Select(q => q.Variance()).ToArray());
      ParameterConfigurationParameter.ActualValue.QualityStandardDeviations = new DoubleArray(qualities.Select(q => q.StandardDeviation()).ToArray());
      ParameterConfigurationParameter.ActualValue.Runs = (RunCollection)algorithm.Runs.Clone();

      // normalize qualities
      double[] qualitiesNormalized = new double[problems.Count];
      for (int i = 0; i < problems.Count; i++) {
        qualitiesNormalized[i] = qualities[i].Average() / medians[i];
      }
      ParameterConfigurationParameter.ActualValue.QualitiesNormalized = new DoubleArray(qualitiesNormalized);
      ParameterConfigurationParameter.ActualValue.AverageQualityNormalized = new DoubleValue(qualitiesNormalized.Average());

      this.QualityParameter.ActualValue = ParameterConfigurationParameter.ActualValue.AverageQualityNormalized;

      return base.Apply();
    }

    public static double Variance(IEnumerable<double> source) {
      double avg = source.Average();
      double d = source.Aggregate(0.0, (total, next) => total += Math.Pow(next - avg, 2));
      return d / (source.Count() - 1);
    }

    public static double StandardDeviation(IEnumerable<double> source) {
      return Math.Sqrt(source.Variance());
    }
  }

  /// <summary>
  /// Can execute an algorithm synchronously
  /// </summary>
  public class AlgorithmExecutor {
    private EngineAlgorithm algorithm;
    private AutoResetEvent waitHandle = new AutoResetEvent(false);

    public AlgorithmExecutor(EngineAlgorithm algorithm) {
      this.algorithm = algorithm;
    }

    public void StartSync() {
      algorithm.Stopped += new EventHandler(algorithm_Stopped);
      algorithm.Paused += new EventHandler(algorithm_Paused);
      algorithm.Start();
      waitHandle.WaitOne();
      waitHandle.Dispose();
      algorithm.Stopped -= new EventHandler(algorithm_Stopped);
      algorithm.Paused -= new EventHandler(algorithm_Paused);
    }

    void algorithm_Paused(object sender, EventArgs e) {
      waitHandle.Set();
    }

    void algorithm_Stopped(object sender, EventArgs e) {
      waitHandle.Set();
    }
  }
}