source: branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/ParameterConfigurationEvaluator.cs @ 5184

using System;
using System.Linq;
using System.Threading;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Operators;
using HeuristicLab.Optimization;
using HeuristicLab.Parameters;
using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
using System.Collections.Generic;

namespace HeuristicLab.Problems.MetaOptimization {
  /// <summary>
  /// A base class for operators which evaluate TSP solutions.
  /// </summary>
  [Item("ParameterConfigurationEvaluator", "A base class for operators which evaluate Meta Optimization solutions.")]
  [StorableClass]
  public class ParameterConfigurationEvaluator : SingleSuccessorOperator, IParameterConfigurationEvaluator {
    private bool algorithmStopped;
    private bool algorithmExceptionOccured;

    public ILookupParameter<DoubleValue> QualityParameter {
      get { return (ILookupParameter<DoubleValue>)Parameters["Quality"]; }
    }
    public ILookupParameter<EngineAlgorithm> AlgorithmParameter {
      get { return (ILookupParameter<EngineAlgorithm>)Parameters[MetaOptimizationProblem.AlgorithmTypeParameterName]; }
    }
    public ILookupParameter<IItemList<ISingleObjectiveProblem>> ProblemsParameter {
      get { return (ILookupParameter<IItemList<ISingleObjectiveProblem>>)Parameters[MetaOptimizationProblem.ProblemsParameterName]; }
    }
    public ILookupParameter<ParameterConfigurationTree> ParameterConfigurationParameter {
      get { return (ILookupParameter<ParameterConfigurationTree>)Parameters["ParameterConfigurationTree"]; }
    }
    public LookupParameter<IntValue> RepetitionsParameter {
      get { return (LookupParameter<IntValue>)Parameters[MetaOptimizationProblem.RepetitionsParameterName]; }
    }

    public IntValue Repetitions {
      get { return RepetitionsParameter.ActualValue; }
    }

    public ParameterConfigurationEvaluator()
      : base() {
      Parameters.Add(new LookupParameter<DoubleValue>("Quality", "The evaluated quality of the ParameterVector."));
      Parameters.Add(new LookupParameter<EngineAlgorithm>(MetaOptimizationProblem.AlgorithmTypeParameterName, "Missing description."));
      Parameters.Add(new LookupParameter<IItemList<ISingleObjectiveProblem>>(MetaOptimizationProblem.ProblemsParameterName, "Missing description."));
      Parameters.Add(new LookupParameter<ParameterConfigurationTree>("ParameterConfigurationTree", "Missing description."));
      Parameters.Add(new LookupParameter<IntValue>(MetaOptimizationProblem.RepetitionsParameterName, "Number of evaluations on one problem."));
    }

    [StorableConstructor]
    protected ParameterConfigurationEvaluator(bool deserializing) : base(deserializing) { }
    protected ParameterConfigurationEvaluator(ParameterConfigurationEvaluator original, Cloner cloner)
      : base(original, cloner) {
      this.algorithmStopped = original.algorithmStopped;
    }
    public override IDeepCloneable Clone(Cloner cloner) {
      return new ParameterConfigurationEvaluator(this, cloner);
    }

    public override IOperation Apply() {
      EngineAlgorithm algorithm = AlgorithmParameter.ActualValue;
      IItemList<ISingleObjectiveProblem> problems = ProblemsParameter.ActualValue;

      // set parameters
      ParameterConfigurationParameter.ActualValue.Parameterize(algorithm);
      algorithm.Problem = problems.First();
      algorithm.StoreAlgorithmInEachRun = false;

      algorithmStopped = false;
      algorithmExceptionOccured = false;
      algorithm.Stopped += new EventHandler(algorithm_Stopped);
      algorithm.Paused += new EventHandler(algorithm_Paused);
      algorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(algorithm_ExceptionOccurred);

      List<double> qualities = new List<double>();
      List<TimeSpan> executionTimes = new List<TimeSpan>();
      algorithm.Engine = new SequentialEngine.SequentialEngine();
      algorithm.Prepare(true);

      foreach (ISingleObjectiveProblem problem in problems) {
        algorithm.Problem = problem;

        for (int i = 0; i < Repetitions.Value; i++) {
          algorithm.Prepare();
          algorithm.Start();
          while (!algorithmStopped) {
            Thread.Sleep(200); // wait for algorithm to complete; do not rely on Algorithm.ExecutionState here, because change of ExecutionState happens before Run is added (which causes problems because Algorithm might get cloned when its started already)
          }

          if (algorithmExceptionOccured) {
            // this parametercombination was bad. set penalty for this solution
            qualities.Add(double.MaxValue); // todo: respect Maximization
            executionTimes.Add(algorithm.ExecutionTime);
          } else {
            qualities.Add(((DoubleValue)algorithm.Results["BestQuality"].Value).Value);
            executionTimes.Add(algorithm.ExecutionTime);

            // parameters will be stored in ParameterConfigurationTree anyway. they would be redundant in runs
            algorithm.Runs.Last().Parameters.Clear();
            // but keep the problem, since this differs in runs
            algorithm.Runs.Last().Parameters.Add("Problem", problem);
          }
          algorithmStopped = false;
          algorithmExceptionOccured = false;
        }
      }

      algorithm.Stopped -= new EventHandler(algorithm_Stopped);
      algorithm.Paused -= new EventHandler(algorithm_Paused);
      algorithm.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(algorithm_ExceptionOccurred);
      algorithm.Prepare();

      qualities = qualities.OrderBy(x => x).ToList();  // todo: respect Maximization:true/false

      ParameterConfigurationParameter.ActualValue.AverageExecutionTime = new TimeSpanValue(TimeSpan.FromMilliseconds(executionTimes.Average(t => t.TotalMilliseconds)));
      ParameterConfigurationParameter.ActualValue.Repetitions = (IntValue)Repetitions.Clone();
      ParameterConfigurationParameter.ActualValue.BestQuality = new DoubleValue(qualities.First());
      ParameterConfigurationParameter.ActualValue.AverageQuality = new DoubleValue(qualities.Average());
      ParameterConfigurationParameter.ActualValue.WorstQuality = new DoubleValue(qualities.Last());
      ParameterConfigurationParameter.ActualValue.QualityVariance = new DoubleValue(qualities.Variance());
      ParameterConfigurationParameter.ActualValue.QualityStandardDeviation = new DoubleValue(qualities.StandardDeviation());
      ParameterConfigurationParameter.ActualValue.Runs = (RunCollection)algorithm.Runs.Clone();

      double quality = ParameterConfigurationParameter.ActualValue.AverageQuality.Value; // todo: also include other measures (executiontime, variance)
      this.QualityParameter.ActualValue = new DoubleValue(quality);

      return base.Apply();
    }

    private void algorithm_Paused(object sender, EventArgs e) {
      algorithmStopped = true;
    }

    private void algorithm_Stopped(object sender, EventArgs e) {
      algorithmStopped = true;
    }

    void algorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
      algorithmExceptionOccured = true;
    }

    public static double Variance(IEnumerable<double> source) {
      double avg = source.Average();
      double d = source.Aggregate(0.0, (total, next) => total += Math.Pow(next - avg, 2));
      return d / (source.Count() - 1);
    }

    public static double StandardDeviation(IEnumerable<double> source) {
      return Math.Sqrt(source.Variance());
    }


  }
}