source: branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/ParameterConfigurationEvaluator.cs @ 5207

using System;
using System.Linq;
using System.Threading;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Operators;
using HeuristicLab.Optimization;
using HeuristicLab.Parameters;
using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
using System.Collections.Generic;
using HeuristicLab.Algorithms.GeneticAlgorithm;
using System.Threading.Tasks;
using System.Diagnostics;
using System.Reflection;

namespace HeuristicLab.Problems.MetaOptimization {
  /// <summary>
  /// A base class for operators which evaluate TSP solutions.
  /// </summary>
  [Item("ParameterConfigurationEvaluator", "A base class for operators which evaluate Meta Optimization solutions.")]
  [StorableClass]
  public class ParameterConfigurationEvaluator : SingleSuccessorOperator, IParameterConfigurationEvaluator {
    private const double PenaltyQuality = 100000.0; // todo: use something better

    public ILookupParameter<DoubleValue> QualityParameter {
      get { return (ILookupParameter<DoubleValue>)Parameters["Quality"]; }
    }
    public ILookupParameter<EngineAlgorithm> AlgorithmParameter {
      get { return (ILookupParameter<EngineAlgorithm>)Parameters[MetaOptimizationProblem.AlgorithmTypeParameterName]; }
    }
    public ILookupParameter<IItemList<ISingleObjectiveProblem>> ProblemsParameter {
      get { return (ILookupParameter<IItemList<ISingleObjectiveProblem>>)Parameters[MetaOptimizationProblem.ProblemsParameterName]; }
    }
    public ILookupParameter<ParameterConfigurationTree> ParameterConfigurationParameter {
      get { return (ILookupParameter<ParameterConfigurationTree>)Parameters["ParameterConfigurationTree"]; }
    }
    public LookupParameter<IntValue> RepetitionsParameter {
      get { return (LookupParameter<IntValue>)Parameters[MetaOptimizationProblem.RepetitionsParameterName]; }
    }

    public IntValue Repetitions {
      get { return RepetitionsParameter.ActualValue; }
    }

    public ParameterConfigurationEvaluator()
      : base() {
      Parameters.Add(new LookupParameter<DoubleValue>("Quality", "The evaluated quality of the ParameterVector."));
      Parameters.Add(new LookupParameter<EngineAlgorithm>(MetaOptimizationProblem.AlgorithmTypeParameterName, "Missing description."));
      Parameters.Add(new LookupParameter<IItemList<ISingleObjectiveProblem>>(MetaOptimizationProblem.ProblemsParameterName, "Missing description."));
      Parameters.Add(new LookupParameter<ParameterConfigurationTree>("ParameterConfigurationTree", "Missing description."));
      Parameters.Add(new LookupParameter<IntValue>(MetaOptimizationProblem.RepetitionsParameterName, "Number of evaluations on one problem."));
    }

    [StorableConstructor]
    protected ParameterConfigurationEvaluator(bool deserializing) : base(deserializing) { }
    protected ParameterConfigurationEvaluator(ParameterConfigurationEvaluator original, Cloner cloner)
      : base(original, cloner) {
    }
    public override IDeepCloneable Clone(Cloner cloner) {
      return new ParameterConfigurationEvaluator(this, cloner);
    }

    public override IOperation Apply() {
      EngineAlgorithm algorithm = (EngineAlgorithm)AlgorithmParameter.ActualValue.Clone();
      IItemList<ISingleObjectiveProblem> problems = ProblemsParameter.ActualValue;

      // set parameters
      ParameterConfigurationParameter.ActualValue.Parameterize(algorithm);
      algorithm.StoreAlgorithmInEachRun = false;

      List<double> qualities = new List<double>();
      List<TimeSpan> executionTimes = new List<TimeSpan>();
      algorithm.Engine = new SequentialEngine.SequentialEngine();
      algorithm.Prepare(true);

      foreach (ISingleObjectiveProblem problem in problems) {
        algorithm.Problem = (IProblem)problem.Clone();

        for (int i = 0; i < Repetitions.Value; i++) {
          algorithm.Prepare();

          AlgorithmExecutor executor = new AlgorithmExecutor(algorithm);
          executor.StartSync();

          if (algorithm.ExecutionState == ExecutionState.Paused) {
            // this parametercombination was bad. set penalty for this solution
            qualities.Add(PenaltyQuality); // todo: respect Maximization; problem: this messes up average value; solution: use currently worst quality*2
            executionTimes.Add(algorithm.ExecutionTime);
          } else {
            qualities.Add(((DoubleValue)algorithm.Results["BestQuality"].Value).Value);
            executionTimes.Add(algorithm.ExecutionTime);

            // parameters will be stored in ParameterConfigurationTree anyway. they would be redundant in runs
            algorithm.Runs.Last().Parameters.Clear();
            // but keep the problem, since this differs in runs
            algorithm.Runs.Last().Parameters.Add("Problem", problem);
          }
        }
      }
      algorithm.Prepare();

      qualities = qualities.OrderBy(x => x).ToList();  // todo: respect Maximization:true/false

      ParameterConfigurationParameter.ActualValue.AverageExecutionTime = new TimeSpanValue(TimeSpan.FromMilliseconds(executionTimes.Average(t => t.TotalMilliseconds)));
      ParameterConfigurationParameter.ActualValue.Repetitions = (IntValue)Repetitions.Clone();
      ParameterConfigurationParameter.ActualValue.BestQuality = new DoubleValue(qualities.First());
      ParameterConfigurationParameter.ActualValue.AverageQuality = new DoubleValue(qualities.Average());
      ParameterConfigurationParameter.ActualValue.WorstQuality = new DoubleValue(qualities.Last());
      ParameterConfigurationParameter.ActualValue.QualityVariance = new DoubleValue(qualities.Variance());
      ParameterConfigurationParameter.ActualValue.QualityStandardDeviation = new DoubleValue(qualities.StandardDeviation());
      ParameterConfigurationParameter.ActualValue.Runs = (RunCollection)algorithm.Runs.Clone();

      double quality = ParameterConfigurationParameter.ActualValue.AverageQuality.Value; // todo: also include other measures (executiontime, variance)
      this.QualityParameter.ActualValue = new DoubleValue(quality);

      return base.Apply();
    }

    public static double Variance(IEnumerable<double> source) {
      double avg = source.Average();
      double d = source.Aggregate(0.0, (total, next) => total += Math.Pow(next - avg, 2));
      return d / (source.Count() - 1);
    }

    public static double StandardDeviation(IEnumerable<double> source) {
      return Math.Sqrt(source.Variance());
    }
  }

  public class AlgorithmExecutor {
    private EngineAlgorithm algorithm;
    private AutoResetEvent waitHandle = new AutoResetEvent(false);

    public AlgorithmExecutor(EngineAlgorithm algorithm) {
      this.algorithm = algorithm;
    }

    public void StartSync() {
      algorithm.Stopped += new EventHandler(algorithm_Stopped);
      algorithm.Paused += new EventHandler(algorithm_Paused);
      algorithm.Start();
      waitHandle.WaitOne();
      waitHandle.Dispose();
      algorithm.Stopped -= new EventHandler(algorithm_Stopped);
      algorithm.Paused -= new EventHandler(algorithm_Paused);
    }

    void algorithm_Paused(object sender, EventArgs e) {
      waitHandle.Set();
    }

    void algorithm_Stopped(object sender, EventArgs e) {
      waitHandle.Set();
    }
  }
}