source: branches/HeuristicLab.Classification/HeuristicLab.Algorithms.DataAnalysis/3.3/CrossValidation.cs @ 4542

#region License Information
/* HeuristicLab
 * Copyright (C) 2002-2010 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */
#endregion

using System;
using System.Drawing;
using System.Linq;
using HeuristicLab.Collections;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Optimization;
using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
using HeuristicLab.Problems.DataAnalysis;

namespace HeuristicLab.Algorithms.DataAnalysis {
  [Item("Cross Validation", "Cross Validation wrapper for data analysis algorithms.")]
  [Creatable("Data Analysis")]
  [StorableClass]
  public sealed class CrossValidation : NamedItem, IOptimizer {
    public CrossValidation()
      : base() {
      name = ItemName;
      description = ItemDescription;

      executionState = ExecutionState.Stopped;
      executionTime = TimeSpan.Zero;
      runs = new RunCollection();

      algorithm = null;
      clonedAlgorithms = new ItemCollection<IAlgorithm>();
      readOnlyClonedAlgorithms = null;

      folds = new IntValue(1);
      numberOfWorkers = new IntValue(1);
      samplesStart = new IntValue(0);
      samplesEnd = new IntValue(0);


      RegisterEvents();
    }

    #region persistence and cloning
    [StorableConstructor]
    private CrossValidation(bool deserializing)
      : base(deserializing) {
    }
    [StorableHook(HookType.AfterDeserialization)]
    private void AfterDeserialization() {
      RegisterEvents();
    }

    public override IDeepCloneable Clone(Cloner cloner) {
      if (ExecutionState == ExecutionState.Started) throw new InvalidOperationException(string.Format("Clone not allowed in execution state \"{0}\".", ExecutionState));
      CrossValidation clone = new CrossValidation(false);
      cloner.RegisterClonedObject(this, clone);
      clone.name = name;
      clone.description = description;
      clone.executionState = executionState;
      clone.executionTime = executionTime;
      clone.runs = (RunCollection)cloner.Clone(runs);
      clone.algorithm = (IAlgorithm)cloner.Clone(algorithm);
      clone.clonedAlgorithms = (ItemCollection<IAlgorithm>)cloner.Clone(clonedAlgorithms);
      clone.folds = (IntValue)cloner.Clone(folds);
      clone.numberOfWorkers = (IntValue)cloner.Clone(numberOfWorkers);
      clone.samplesStart = (IntValue)cloner.Clone(samplesStart);
      clone.samplesEnd = (IntValue)cloner.Clone(samplesEnd);
      clone.RegisterEvents();
      return clone;
    }

    #endregion

    #region properties
    [Storable]
    private IAlgorithm algorithm;
    public IAlgorithm Algorithm {
      get { return algorithm; }
      set {
        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
          throw new InvalidOperationException("Changing the algorithm is only allowed if the CrossValidation is stopped or prepared.");
        if (algorithm != value) {
          if (value != null && value.Problem != null && !(value.Problem is IDataAnalysisProblem))
            throw new ArgumentException("Only algorithms with a DataAnalysisProblem could be used for the cross validation.");
          if (algorithm != null) DeregisterAlgorithmEvents();
          algorithm = value;

          if (algorithm != null) {
            RegisterAlgorithmEvents();
            algorithm.Prepare(true);
          }
          OnAlgorithmChanged();
          Prepare();
        }
      }
    }

    [Storable]
    private IDataAnalysisProblem cachedProblem;
    public IDataAnalysisProblem Problem {
      get {
        if (algorithm == null)
          return null;
        return (IDataAnalysisProblem)algorithm.Problem;
      }
      set {
        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
          throw new InvalidOperationException("Changing the problem is only allowed if the CrossValidation is stopped or prepared.");
        if (algorithm == null) throw new ArgumentNullException("Could not set a problem before an algorithm was set.");
        algorithm.Problem = value;
        cachedProblem = value;
      }
    }

    [Storable]
    private ItemCollection<IAlgorithm> clonedAlgorithms;
    private ReadOnlyItemCollection<IAlgorithm> readOnlyClonedAlgorithms;
    public IItemCollection<IAlgorithm> ClonedAlgorithms {
      get {
        if (readOnlyClonedAlgorithms == null) readOnlyClonedAlgorithms = clonedAlgorithms.AsReadOnly();
        return readOnlyClonedAlgorithms;
      }
    }

    [Storable]
    private IntValue folds;
    public IntValue Folds {
      get { return folds; }
    }
    [Storable]
    private IntValue samplesStart;
    public IntValue SamplesStart {
      get { return samplesStart; }
    }
    [Storable]
    private IntValue samplesEnd;
    public IntValue SamplesEnd {
      get { return samplesEnd; }
    }
    [Storable]
    private IntValue numberOfWorkers;
    public IntValue NumberOfWorkers {
      get { return numberOfWorkers; }
    }

    [Storable]
    private RunCollection runs;
    public RunCollection Runs {
      get { return runs; }
    }
    [Storable]
    private ExecutionState executionState;
    public ExecutionState ExecutionState {
      get { return executionState; }
      private set {
        if (executionState != value) {
          executionState = value;
          OnExecutionStateChanged();
          OnItemImageChanged();
        }
      }
    }
    public override Image ItemImage {
      get {
        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VS2008ImageLibrary.ExecutablePrepared;
        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VS2008ImageLibrary.ExecutableStarted;
        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VS2008ImageLibrary.ExecutablePaused;
        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VS2008ImageLibrary.ExecutableStopped;
        else return HeuristicLab.Common.Resources.VS2008ImageLibrary.Event;
      }
    }

    [Storable]
    private TimeSpan executionTime;
    public TimeSpan ExecutionTime {
      get {
        if (ExecutionState != ExecutionState.Stopped)
          return executionTime + TimeSpan.FromMilliseconds(clonedAlgorithms.Select(x => x.ExecutionTime.TotalMilliseconds).Sum());
        else
          return executionTime;
      }
      private set {
        executionTime = value;
        OnExecutionTimeChanged();
      }
    }
    #endregion

    public void Prepare() {
      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused) && (ExecutionState != ExecutionState.Stopped))
        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
      clonedAlgorithms.Clear();
      if (Algorithm != null) {
        Algorithm.Prepare();
        if (Algorithm.ExecutionState == ExecutionState.Prepared) OnPrepared();
      }
    }
    public void Prepare(bool clearRuns) {
      if (clearRuns) runs.Clear();
      Prepare();
    }

    private bool startPending;
    public void Start() {
      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
        throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));

      if (Algorithm != null && !startPending) {
        startPending = true;
        //create cloned algorithms
        if (clonedAlgorithms.Count == 0) {
          for (int i = 0; i < Folds.Value; i++) {
            IAlgorithm clonedAlgorithm = (IAlgorithm)algorithm.Clone();
            clonedAlgorithm.Name = algorithm.Name + " Fold " + i;
            clonedAlgorithms.Add(clonedAlgorithm);
          }
        }

        //start prepared or paused cloned algorithms
        int startedAlgorithms = 0;
        foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
          if (startedAlgorithms < NumberOfWorkers.Value) {
            if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared ||
                clonedAlgorithm.ExecutionState == ExecutionState.Paused) {
              clonedAlgorithm.Start();
              startedAlgorithms++;
            }
          }
        }
        OnStarted();
      }
    }

    private bool pausePending;
    public void Pause() {
      if (ExecutionState != ExecutionState.Started)
        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
      if (!pausePending) {
        pausePending = true;
        if (!startPending) PauseAllClonedAlgorithms();
      }
    }
    private void PauseAllClonedAlgorithms() {
      foreach (IAlgorithm clonedAlgorithm in ClonedAlgorithms) {
        if (clonedAlgorithm.ExecutionState == ExecutionState.Started)
          clonedAlgorithm.Pause();
      }
    }

    private bool stopPending;
    public void Stop() {
      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".",
                                                          ExecutionState));
      if (!stopPending) {
        stopPending = true;
        if (!startPending) StopAllClonedAlgorithms();
      }
    }
    private void StopAllClonedAlgorithms() {
      foreach (IAlgorithm clonedAlgorithm in ClonedAlgorithms) {
        if (clonedAlgorithm.ExecutionState == ExecutionState.Started ||
            clonedAlgorithm.ExecutionState == ExecutionState.Paused)
          clonedAlgorithm.Stop();
      }
    }

    #region events
    private void RegisterEvents() {
      Folds.ValueChanged += new EventHandler(Folds_ValueChanged);
      NumberOfWorkers.ValueChanged += new EventHandler(NumberOfWorkers_ValueChanged);
      RegisterClonedAlgorithmsEvents();
      RegisterRunsEvents();
    }
    private void Folds_ValueChanged(object sender, EventArgs e) {
      if (ExecutionState != ExecutionState.Prepared)
        throw new InvalidOperationException("Can not change number of folds if the execution state is not prepared.");
    }
    private void NumberOfWorkers_ValueChanged(object sender, EventArgs e) {
      if (ExecutionState == ExecutionState.Started) {
        int workers = numberOfWorkers.Value;
        int runningWorkers = clonedAlgorithms.Count(alg => alg.ExecutionState == ExecutionState.Started);

        foreach (IAlgorithm algorithm in clonedAlgorithms) {
          if (algorithm.ExecutionState == ExecutionState.Prepared ||
              algorithm.ExecutionState == ExecutionState.Paused) {
            if (runningWorkers < workers) {
              algorithm.Start();
              runningWorkers++;
            }
          } else if (algorithm.ExecutionState == ExecutionState.Started) {
            if (runningWorkers > workers) {
              algorithm.Pause();
              runningWorkers--;
            }
          }
        }
      }
    }

    #region template algorithms events
    public event EventHandler AlgorithmChanged;
    private void OnAlgorithmChanged() {
      EventHandler handler = AlgorithmChanged;
      if (handler != null) handler(this, EventArgs.Empty);
      OnProblemChanged();
      if (Problem == null) ExecutionState = ExecutionState.Stopped;
    }
    private void RegisterAlgorithmEvents() {
      algorithm.ProblemChanged += new EventHandler(Algorithm_ProblemChanged);
      algorithm.ExecutionStateChanged += new EventHandler(Algorithm_ExecutionStateChanged);
    }
    private void DeregisterAlgorithmEvents() {
      algorithm.ProblemChanged -= new EventHandler(Algorithm_ProblemChanged);
      algorithm.ExecutionStateChanged -= new EventHandler(Algorithm_ExecutionStateChanged);
    }
    private void Algorithm_ProblemChanged(object sender, EventArgs e) {
      if (algorithm.Problem != null && !(algorithm.Problem is IDataAnalysisProblem)) {
        algorithm.Problem = cachedProblem;
        throw new ArgumentException("A cross validation algorithm can only contain DataAnalysisProblems.");
      }
      cachedProblem = (IDataAnalysisProblem)algorithm.Problem;
      SamplesStart.Value = 0;
      if (algorithm.Problem != null)
        SamplesEnd.Value = Problem.DataAnalysisProblemData.Dataset.Rows;
      else
        SamplesEnd.Value = 0;
      OnProblemChanged();
    }
    public event EventHandler ProblemChanged;
    private void OnProblemChanged() {
      EventHandler handler = ProblemChanged;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
      switch (Algorithm.ExecutionState) {
        case ExecutionState.Prepared: OnPrepared();
          break;
        case ExecutionState.Started: throw new InvalidOperationException("Algorithm template can not be started.");
        case ExecutionState.Paused: throw new InvalidOperationException("Algorithm template can not be paused.");
        case ExecutionState.Stopped: OnStopped();
          break;
      }
    }
    #endregion

    #region clonedAlgorithms events
    private void RegisterClonedAlgorithmsEvents() {
      clonedAlgorithms.ItemsAdded += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
      clonedAlgorithms.ItemsRemoved += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
      clonedAlgorithms.CollectionReset += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
      foreach (IAlgorithm algorithm in clonedAlgorithms)
        RegisterClonedAlgorithmEvents(algorithm);
    }
    private void ClonedAlgorithms_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
      foreach (IAlgorithm algorithm in e.Items)
        RegisterClonedAlgorithmEvents(algorithm);
    }
    private void ClonedAlgorithms_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
      foreach (IAlgorithm algorithm in e.Items)
        DeregisterClonedAlgorithmEvents(algorithm);
    }
    private void ClonedAlgorithms_CollectionReset(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
      foreach (IAlgorithm algorithm in e.OldItems)
        DeregisterClonedAlgorithmEvents(algorithm);
      foreach (IAlgorithm algorithm in e.Items)
        RegisterClonedAlgorithmEvents(algorithm);
    }
    private void RegisterClonedAlgorithmEvents(IAlgorithm algorithm) {
      algorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
      algorithm.ExecutionTimeChanged += new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
      algorithm.Started += new EventHandler(ClonedAlgorithm_Started);
      algorithm.Paused += new EventHandler(ClonedAlgorithm_Paused);
      algorithm.Stopped += new EventHandler(ClonedAlgorithm_Stopped);
    }
    private void DeregisterClonedAlgorithmEvents(IAlgorithm algorithm) {
      algorithm.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
      algorithm.ExecutionTimeChanged -= new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
      algorithm.Started -= new EventHandler(ClonedAlgorithm_Started);
      algorithm.Paused -= new EventHandler(ClonedAlgorithm_Paused);
      algorithm.Stopped -= new EventHandler(ClonedAlgorithm_Stopped);
    }
    private void ClonedAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
      OnExceptionOccurred(e.Value);
    }
    private void ClonedAlgorithm_ExecutionTimeChanged(object sender, EventArgs e) {
      OnExecutionTimeChanged();
    }

    private readonly object locker = new object();
    private void ClonedAlgorithm_Started(object sender, EventArgs e) {
      lock (locker) {
        if (startPending) {
          int startedAlgorithms = clonedAlgorithms.Count(alg => alg.ExecutionState == ExecutionState.Started);
          if (startedAlgorithms == NumberOfWorkers.Value ||
             clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Prepared))
            startPending = false;

          if (pausePending) PauseAllClonedAlgorithms();
          if (stopPending) StopAllClonedAlgorithms();
        }
      }
    }

    private void ClonedAlgorithm_Paused(object sender, EventArgs e) {
      lock (locker) {
        if (pausePending && clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started))
          OnPaused();
      }
    }

    private void ClonedAlgorithm_Stopped(object sender, EventArgs e) {
      lock (locker) {
        if (!stopPending && ExecutionState == ExecutionState.Started) {
          IAlgorithm preparedAlgorithm = clonedAlgorithms.Where(alg => alg.ExecutionState == ExecutionState.Prepared ||
                                                                       alg.ExecutionState == ExecutionState.Paused).FirstOrDefault();
          if (preparedAlgorithm != null) preparedAlgorithm.Start();
        }
        if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped))
          OnStopped();
        else if (stopPending && clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped))
          OnStopped();
      }
    }
    #endregion

    #region run events
    private void RegisterRunsEvents() {
      runs.CollectionReset += new CollectionItemsChangedEventHandler<IRun>(Runs_CollectionReset);
      runs.ItemsAdded += new CollectionItemsChangedEventHandler<IRun>(Runs_ItemsAdded);
      runs.ItemsRemoved += new CollectionItemsChangedEventHandler<IRun>(Runs_ItemsRemoved);
    }
    private void Runs_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
      foreach (IRun run in e.OldItems) {
        IItem item;
        run.Results.TryGetValue("Execution Time", out item);
        TimeSpanValue executionTime = item as TimeSpanValue;
        if (executionTime != null) ExecutionTime -= executionTime.Value;
      }
      foreach (IRun run in e.Items) {
        IItem item;
        run.Results.TryGetValue("Execution Time", out item);
        TimeSpanValue executionTime = item as TimeSpanValue;
        if (executionTime != null) ExecutionTime += executionTime.Value;
      }
    }
    private void Runs_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IRun> e) {
      foreach (IRun run in e.Items) {
        IItem item;
        run.Results.TryGetValue("Execution Time", out item);
        TimeSpanValue executionTime = item as TimeSpanValue;
        if (executionTime != null) ExecutionTime += executionTime.Value;
      }
    }
    private void Runs_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IRun> e) {
      foreach (IRun run in e.Items) {
        IItem item;
        run.Results.TryGetValue("Execution Time", out item);
        TimeSpanValue executionTime = item as TimeSpanValue;
        if (executionTime != null) ExecutionTime -= executionTime.Value;
      }
    }
    #endregion
    #endregion

    #region event firing
    public event EventHandler ExecutionStateChanged;
    private void OnExecutionStateChanged() {
      EventHandler handler = ExecutionStateChanged;
      if (handler != null) handler(this, EventArgs.Empty);
    }
    public event EventHandler ExecutionTimeChanged;
    private void OnExecutionTimeChanged() {
      EventHandler handler = ExecutionTimeChanged;
      if (handler != null) handler(this, EventArgs.Empty);
    }
    public event EventHandler Prepared;
    private void OnPrepared() {
      ExecutionState = ExecutionState.Prepared;
      EventHandler handler = Prepared;
      if (handler != null) handler(this, EventArgs.Empty);
    }
    public event EventHandler Started;
    private void OnStarted() {
      startPending = false;
      ExecutionState = ExecutionState.Started;
      EventHandler handler = Started;
      if (handler != null) handler(this, EventArgs.Empty);
    }
    public event EventHandler Paused;
    private void OnPaused() {
      pausePending = false;
      ExecutionState = ExecutionState.Paused;
      EventHandler handler = Paused;
      if (handler != null) handler(this, EventArgs.Empty);
    }
    public event EventHandler Stopped;
    private void OnStopped() {
      stopPending = false;
      ExecutionState = ExecutionState.Stopped;
      //TODO create run;
      EventHandler handler = Stopped;
      if (handler != null) handler(this, EventArgs.Empty);
    }
    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
    private void OnExceptionOccurred(Exception exception) {
      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
      if (handler != null) handler(this, new EventArgs<Exception>(exception));
    }
    #endregion
  }
}