source: branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4/DatastreamAnalysisOptimizer.cs @ 14710

#region License Information

/* HeuristicLab
 * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */

#endregion

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.Drawing;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using HeuristicLab.Analysis;
using HeuristicLab.Collections;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Optimization;
using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
using HeuristicLab.Problems.DataAnalysis;

namespace HeuristicLab.DatastreamAnalysis {
  internal enum DatastreamAnalysisOptimizerAction {
    None,
    Prepare,
    Start,
    Stop,
    Pause
  }

  [StorableClass]
  [Item("DatastreamAnalysis Optimizer",
     "The main loop for evaluating ensemble models against a incoming datastream of time series fashion.")]
  [Creatable(CreatableAttribute.Categories.Algorithms)]
  public class DatastreamAnalysisOptimizer : Executable, IOptimizer, IStorableContent {
    #region properties
    public string Filename { get; set; }

    private DatastreamAnalysisOptimizerAction daoAction;

    public IEnumerable<IOptimizer> NestedOptimizers { get; }


    [Storable]
    protected ILog log;
    public ILog Log {
      get { return log; }
    }

    [Storable]
    private ResultCollection results;

    public ResultCollection Results {
      get { return results; }
    }

    private double lastStateValue;
    private double pR2Sum;
    private int detectionCount;
    private int evaluationCount;
    private double exclusivitySum;
    private double accuracySum;

    private CancellationTokenSource cancellationTokenSource;
    private bool stopPending;
    private DateTime lastUpdateTime;
    private bool prepared;
    private bool finished;

    [Storable]
    protected int runsCounter;

    [Storable]
    private RunCollection runs;

    public RunCollection Runs
    {
      get { return runs; }
      protected set
      {
        if (value == null) throw new ArgumentNullException();
        if (runs != value) {
          if (runs != null) DeregisterRunsEvents();
          runs = value;
          if (runs != null) RegisterRunsEvents();
        }
      }
    }

    [Storable]
    private IItemList<RatedEnsembleModel> ensembles;

    public IItemList<RatedEnsembleModel> Ensembles {
      get { return ensembles; }
      set {
        if (value == null || value == ensembles)
          return;
        if (!(value is IRatedEnsembleModel)) throw new ArgumentException("Invaid ensemble model type");
        DeregisterEnsembleEvents();
        ensembles = value;
        RegisterEnsembleEvents();
        OnEnsemblesChanged();
        Prepare();
      }
    }


    // VAR 1: datastream ~= problem data, VAR 2 (TODO): datastream = external source e.g. webservice, AMQP-Queue, etc.
    [Storable]
    private Datastream datastream;

    public Datastream Datastream {
      get { return datastream; }
      set {
        if (value == null || value == datastream)
          return;
        if(!(value is IDatastream)) throw new ArgumentException("Invalid datastream type");
        DeregisterDatastreamEvents();
        datastream = value;
        RegisterDatastreamEvents();
        OnDatastreamChanged();
        Prepare();
      }
    }

    #endregion properties

    #region results properties
    private int ResultsSlidingWindowMovements {
      get { return ((IntValue)Results["Sliding Window Movements"].Value).Value; }
      set { ((IntValue)Results["Sliding Window Movements"].Value).Value = value; }
    }
    private DataTable ResultsQualities {
      get { return ((DataTable)Results["Qualities 1"].Value); }
    }

    private DataTable ResultsPeakQualities {
      get { return ((DataTable) Results["Qualities 2"].Value); }
    }

    private const string ResultsQualitiesMSE = "Mean squared error";
    private const string ResultsQualitiesPR2 = "Pearson R²";

    private DataTable ResultsTargets {
      get { return ((DataTable) Results["Targets"].Value); }
    }

    private DataTable ResultsStateDetection {
      get { return ((DataTable) Results["State Detection"].Value);}
    }

    private const string ResultsTargetsReal = "Real";

    private DataBarSet ResultsSWQualitiesBars {
      get { return (DataBarSet) Results["Ensemble Comparison [SW-Quality]"].Value; }
    }

    private DataBarSet ResultsSWVotingBars {
      get { return (DataBarSet) Results["Ensemble Comparison [SW-Voting]"].Value; }
    }

    private DataBarSet ResultsSonarQualitiesBars {
      get { return (DataBarSet)Results["Ensemble Comparison [Sonar-Quality]"].Value; }
    }

    private DataBarSet ResultsSWPeakQualitiesBars {
      get { return (DataBarSet)Results["Ensemble Comparison [SW-PeakQuality]"].Value; }
    }

    private double ResultsMeanPR2 {
      get { return ((DoubleValue) Results["Mean PR2"].Value).Value; }
      set { ((DoubleValue)Results["Mean PR2"].Value).Value = value; }
    }

    private double ResultsDetectionAccuracy {
      get { return ((DoubleValue)Results["Detection Accuracy"].Value).Value; }
      set { ((DoubleValue)Results["Detection Accuracy"].Value).Value = value; }
    }

    private double ResultsDetectionExclusivity {
      get { return ((DoubleValue)Results["Detection Exclusivity"].Value).Value; }
      set { ((DoubleValue)Results["Detection Exclusivity"].Value).Value = value; }
    }

    //private int ResultsTrueDetectionCount {
    //  get { return ((IntValue)Results["True Detection Count"].Value).Value; }
    //  set { ((IntValue)Results["True Detection Count"].Value).Value = value; }
    //}

    //private int ResultsFalseDetectionCount {
    //  get { return ((IntValue)Results["False Detection Count"].Value).Value; }
    //  set { ((IntValue)Results["False Detection Count"].Value).Value = value; }
    //}

    protected void SetupResults() {
      evaluationCount = 0;
      lastStateValue = 0.0;
      pR2Sum = 0.0;
      exclusivitySum = 0.0;
      accuracySum = 0.0;
      detectionCount = 0;
      Results.Clear();
      Results.Add(new Result("Sliding Window Movements", new IntValue(0)));
      Results.Add(new Result("Qualities 1", new DataTable("Average Pearson R²")));
      Results.Add(new Result("Qualities 2", new DataTable("Peak Pearson R²")));
      Results.Add(new Result("Targets", new DataTable("Targets")));
      Results.Add(new Result("State Detection", new DataTable("State Detection")));
      Results.Add(new Result("Ensemble Comparison [SW-Quality]", new DataBarSet("Ensemble Comparison [SW-Quality]")));
      Results.Add(new Result("Ensemble Comparison [SW-Voting]", new DataBarSet("Ensemble Comparison [SW-Voting]")));
      Results.Add(new Result("Ensemble Comparison [Sonar-Quality]", new DataBarSet("Ensemble Comparison [Sonar-Quality]")));
      Results.Add(new Result("Ensemble Comparison [SW-PeakQuality]", new DataBarSet("Ensemble Comparison [SW-PeakQuality]")));
      Results.Add(new Result("Mean PR2", new DoubleValue()));
      Results.Add(new Result("Detection Accuracy", new DoubleValue()));
      Results.Add(new Result("Detection Exclusivity", new DoubleValue()));

      ResultsTargets.Rows.Add(new DataRow(ResultsTargetsReal));
      foreach (var ensemble in Ensembles) {
        // targets table
        ResultsTargets.Rows.Add(new DataRow(ensemble.Name));

        ResultsStateDetection.Rows.Add(new DataRow(ensemble.Name));

        // qualities (series)
        //ResultsQualities.Rows.Add(new DataRow(ensemble.Name + " - " + ResultsQualitiesMSE));
        ResultsQualities.Rows.Add(new DataRow(ensemble.Name));

        ResultsPeakQualities.Rows.Add(new DataRow(ensemble.Name));

        // qualities (bars)
        ResultsSWQualitiesBars.Bars.Add(new DataBar(ensemble.Name, ensemble.QualityThreshold.Start, ensemble.QualityThreshold.End));

        // voting (bars)
        ResultsSWVotingBars.Bars.Add(new DataBar(ensemble.Name, ensemble.ConfidenceThreshold.Start, ensemble.ConfidenceThreshold.End));

        // sonar quality (bars)
        ResultsSonarQualitiesBars.Bars.Add(new DataBar(ensemble.Name, ensemble.QualityThreshold.Start, ensemble.QualityThreshold.End));

        // quality peaks (bars)
        ResultsSWPeakQualitiesBars.Bars.Add(new DataBar(ensemble.Name, ensemble.QualityThreshold.Start, ensemble.QualityThreshold.End));
      }
    }

    #endregion

    #region constructors, cloner,...
    public DatastreamAnalysisOptimizer() : base() {
      name = "Datastream Analysis";
      log = new Log();
      results = new ResultCollection();
      ensembles = new ItemList<RatedEnsembleModel>();
      datastream = new Datastream();
      runsCounter = 0;
      runs = new RunCollection();
      Initialize();
    }

    [StorableConstructor]
    protected DatastreamAnalysisOptimizer(bool deserializing) : base(deserializing) {
    }

    [StorableHook(HookType.AfterDeserialization)]
    private void AfterDeserialization() {
      Initialize();
    }

    protected DatastreamAnalysisOptimizer(DatastreamAnalysisOptimizer original, Cloner cloner) : base(original, cloner) {
      name = original.name;
      log = cloner.Clone(original.log);
      results = cloner.Clone(original.results);
      ensembles = (ItemList<RatedEnsembleModel>) original.Ensembles.Clone(cloner);
      datastream = (Datastream) original.Datastream.Clone(cloner);
      runsCounter = original.runsCounter;
      runs = cloner.Clone(original.runs);
      Initialize();
    }

    public override IDeepCloneable Clone(Cloner cloner) {
      return new DatastreamAnalysisOptimizer(this, cloner);
    }

    private void Initialize() {
      if (runs != null) RegisterRunsEvents();
      if (datastream != null) RegisterDatastreamEvents();
      if (ensembles != null) RegisterEnsembleEvents();
    }
    #endregion

    #region control actions

    public override void Prepare() {
      if (ensembles == null || ensembles.Count == 0 || datastream == null || !datastream.SlidingWindowEvaluationPossible) return;
      //if (ensembles.SelectMany(x => x.Models).Count() == 0) return;
      base.Prepare();
      OnPrepared();
    }

    public void Prepare(bool clearRuns) {
      if (ensembles == null || ensembles.Count == 0 || datastream == null || !datastream.SlidingWindowEvaluationPossible) return;

      base.Prepare();
      if (clearRuns) runs.Clear();
      OnPrepared();
    }

    public override void Start() {
      if (ensembles == null || datastream == null) return;
      base.Start();
      cancellationTokenSource = new CancellationTokenSource();
      stopPending = false;

      if (prepared) {
        SetupResults();
        Datastream.InitializeState();
      }

      Task task = Task.Factory.StartNew(Run, cancellationTokenSource.Token, cancellationTokenSource.Token);
      task.ContinueWith(t => {
        try {
          t.Wait();
        }
        catch (AggregateException ex) {
          try {
            ex.Flatten().Handle(x => x is OperationCanceledException);
          } catch (AggregateException remaining) {
            if(remaining.InnerExceptions.Count == 1) OnExceptionOccurred(remaining.InnerExceptions[0]);
            else OnExceptionOccurred(remaining);
          }
        }
        cancellationTokenSource.Dispose();
        cancellationTokenSource = null;

        // handle stop/pause
        if (stopPending || finished) {
          OnStopped();
        } else {
          OnPaused();
        }
      });
    }

    public override void Pause() {
      if (ensembles == null || datastream == null) return;
      base.Pause();
      cancellationTokenSource.Cancel();
    }

    public override void Stop() {
      if (ensembles == null || datastream == null) return;
      base.Stop();
      if (ExecutionState == ExecutionState.Paused) {
        OnStopped();
      } else {
        stopPending = true;
        cancellationTokenSource.Cancel();
      }
    }

    protected override void OnPrepared() {
      ExecutionTime = TimeSpan.Zero;
      foreach (IStatefulItem statefulItem in this.GetObjectGraphObjects(new HashSet<object>() {Runs}).OfType<IStatefulItem>()) {
        statefulItem.InitializeState();
      }
      results.Clear();
      prepared = true;
      finished = false;
      Log.LogMessage("Datastream analysis prepared");
      base.OnPrepared();
    }

    protected override void OnStarted() {
      Log.LogMessage("Datastream analysis started");
      base.OnStarted();
    }

    protected override void OnPaused() {
      Log.LogMessage("Datastream analysis paused");
      base.OnPaused();
    }

    protected override void OnStopped() {
      try {
        runsCounter++;
        var run = new Run();
        run.Filename = Filename;
        run.Name = string.Format("{0} Run {1}", Name, runsCounter);
        CollectParameterValues(run.Parameters);
        CollectResultValues(run.Results);
        runs.Add(run);
      }
      finally {
        Log.LogMessage("Datastream analysis stopped");
        base.OnStopped();
      }
    }

    #endregion

    #region run
    private void Run(object state) {
      CancellationToken cancellationToken = (CancellationToken) state;
      OnStarted();
      lastUpdateTime = DateTime.UtcNow;
      System.Timers.Timer timer = new System.Timers.Timer(250);
      timer.AutoReset = true;
      timer.Elapsed += new System.Timers.ElapsedEventHandler(timer_Elapsed);
      timer.Start();

      try {
        Run(cancellationToken);
      }
      finally {
        timer.Elapsed -= new System.Timers.ElapsedEventHandler(timer_Elapsed);
        timer.Stop();
        ExecutionTime += DateTime.UtcNow - lastUpdateTime;
      }

      cancellationToken.ThrowIfCancellationRequested();
    }

    private int replayedIndex;

    protected void Run(CancellationToken cancellationToken) {

      if (prepared) {
        replayedIndex = 0;
        prepared = false;
      }

      try {

        // play and evaluate initial window
        PlayDatastream();
        lastStateValue = Datastream.ProblemData.Dataset.GetDoubleValue("h", replayedIndex);
        if (Datastream.SlidingWindowEvaluationPossible) Evaluate();
        replayedIndex = Datastream.FitnessPartition.End;

        do {
          while(Datastream.SlidingWindowMovementPossible) {
            cancellationToken.ThrowIfCancellationRequested();

            // perform (delayed) window movement
            Thread.Sleep(Datastream.SlidingWindowMovementDelay.Value);
            Datastream.MoveSlidingWindow();
            ResultsSlidingWindowMovements++;

            // play and evaluate the moved window
            PlayDatastream();
            if (Datastream.SlidingWindowEvaluationPossible) Evaluate();
            replayedIndex = Datastream.FitnessPartition.End;
          }
        } while (Datastream.UpdateAvailable);
        finished = true;
      }
      catch (Exception ex) {
        if (ex is ArgumentOutOfRangeException) throw ex;
        if (ex is OperationCanceledException) throw ex;
      }
      finally {
        // reset everything
        //Prepare(true);
      }
    }

    private void PlayDatastream() {
      var problemData = Datastream.ProblemData;
      var targetVarName = problemData.TargetVariable;

      for (int i = replayedIndex; i < Datastream.FitnessPartition.End; i++) {
        var realValue = problemData.Dataset.GetDoubleValue(targetVarName, i);
        ResultsTargets.Rows[ResultsTargetsReal].Values.Add(realValue);
      }
    }
    
    private void Evaluate() {
      evaluationCount++;
      var problemData = Datastream.ProblemData;
      var dataset = problemData.Dataset;
      var targetVarName = problemData.TargetVariable;

      var rows = Enumerable.Range(Datastream.FitnessPartition.Start, Datastream.FitnessPartition.Size);
      var realValues = dataset.GetDoubleValues(targetVarName, rows);

      int sonarSize = (int)(Datastream.FitnessPartition.Size * Datastream.SlidingWindowSonarRatio.Value);
      var sonarStart = Datastream.FitnessPartition.End - sonarSize;
      var sonarRows = Enumerable.Range(sonarStart, sonarSize);
      var sonarRealValues = dataset.GetDoubleValues(targetVarName, sonarRows);

      var winningEnsembleName = ensembles[0].Name;
      var winningEnsemblePR2 = 0.0;
      var winningEnsembleIdx = 0;
      var ec = 0;
      bool stateChangeInStep = false;
      var curDetectionCount = 0;
      var changeDetectionCount = 0;
      double curStateValue = 0.0;
      foreach (var ensemble in Ensembles) {

        var sonarEstimatedValuesPerModelPerRow = ensemble.Model.Models.Select(x => x.GetEstimatedValues(datastream.ProblemData.Dataset, sonarRows).ToArray());
        var sonarEstimatedValuesPerRow = Enumerable.Range(0, sonarSize).Select(r => sonarEstimatedValuesPerModelPerRow.Select(m => m[r]).Average()).ToArray();

        var estimatedValuesPerModelPerRow = ensemble.Model.Models.Select(x => x.GetEstimatedValues(datastream.ProblemData.Dataset, rows).ToArray());
        var estimatedValuesPerRow = Enumerable.Range(0, Datastream.FitnessPartition.Size).Select(r => estimatedValuesPerModelPerRow.Select(e => e[r]).Average()).ToArray(); // per row
        var averageEstimatedValuesPerModel = estimatedValuesPerModelPerRow.Select(x => x.Average()); // per model
        var averageEstimatedValue = averageEstimatedValuesPerModel.Average();

        // calulate quality
        //var mse = Math.Pow(averageEstimatedValue - realValues.Average(), 2);
        OnlineCalculatorError error;
        var pR = OnlinePearsonsRCalculator.Calculate(estimatedValuesPerRow, realValues, out error);
        var pR2 = error == OnlineCalculatorError.None ? pR * pR : 0.0;

        var sonarPR = OnlinePearsonsRCalculator.Calculate(sonarEstimatedValuesPerRow, sonarRealValues, out error);
        var sonarPR2 = error == OnlineCalculatorError.None ? sonarPR * sonarPR : 0.0;
   
        var estimatedValuesPerModelPerRowArr = estimatedValuesPerModelPerRow.ToArray();
        var curPR = 0.0;
        
        var peakPR = OnlinePearsonsRCalculator.Calculate(estimatedValuesPerModelPerRowArr[0], realValues, out error);
        for (int i = 1; i < estimatedValuesPerModelPerRow.Count(); i++) {
          curPR = OnlinePearsonsRCalculator.Calculate(estimatedValuesPerModelPerRowArr[i], realValues, out error);
          if (curPR > peakPR && error == OnlineCalculatorError.None) {
            peakPR = curPR;
          }
        }
        var peakPR2 = peakPR * peakPR;

        // calculate confidence
        var cm1 = CalulateEnsembleConfidenceCM1(ensemble, estimatedValuesPerModelPerRow, realValues);

        int replayAmount = Datastream.FitnessPartition.End - replayedIndex;
        int replayCount = estimatedValuesPerRow.Length - replayAmount;

        for (int i = replayedIndex; i < Datastream.FitnessPartition.End; i++) {
          //ResultsTargets.Rows[ensemble.Name].Values.Add(averageEstimatedValue);
          ResultsTargets.Rows[ensemble.Name].Values.Add(estimatedValuesPerRow[replayCount]);
          replayCount++;

          //ResultsQualities.Rows[ensemble.Name + " - " + ResultsQualitiesMSE].Values.Add(mse);
          ResultsQualities.Rows[ensemble.Name].Values.Add(pR2);
          ResultsPeakQualities.Rows[ensemble.Name].Values.Add(peakPR2);

          double amp = (pR2 > ensemble.QualityThreshold.Start) ? 1.0 : 0.0;
          ResultsStateDetection.Rows[ensemble.Name].Values.Add(amp);    
        }
        ResultsSWQualitiesBars.Bars[ensemble.Name].Value = pR2;
        ResultsSonarQualitiesBars.Bars[ensemble.Name].Value = sonarPR2;
        ResultsSWVotingBars.Bars[ensemble.Name].Value = cm1;
        ResultsSWPeakQualitiesBars.Bars[ensemble.Name].Value = peakPR2;

        if (pR2 > ensemble.QualityThreshold.Start)
          curDetectionCount++;

        if (pR2 > winningEnsemblePR2) {
          winningEnsemblePR2 = pR2;
          winningEnsembleName = ensemble.Name;
          winningEnsembleIdx = ec;
        }
        ec++;
      }

      // quality meassures
      // =================================================

      if (winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
        // Mean PR2
        pR2Sum += winningEnsemblePR2;
        detectionCount++;
        ResultsMeanPR2 = pR2Sum/detectionCount;

        // detection exclusivity --> 1.0 - (x - 1) * (1.0 / n)
        var fraction = 1.0 / ensembles.Count;
        var curExclusivity = 1.0 - (curDetectionCount - 1) * fraction;
        exclusivitySum += curExclusivity;
        ResultsDetectionExclusivity = exclusivitySum / detectionCount;
      }

      double replCount = 0.0;
      double accCount = 0.0;
      for (int i = replayedIndex; i < Datastream.FitnessPartition.End; i++) {
        replCount++;
        curStateValue = problemData.Dataset.GetDoubleValue("h", i);
        if (curStateValue <= 0.5 && winningEnsembleIdx == 0 && winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
          accCount++;
        } else if (curStateValue > 0.5 && winningEnsembleIdx == 1 && winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
          accCount++;
        }
      }
      accuracySum += accCount / replCount;
      ResultsDetectionAccuracy = accuracySum / evaluationCount;

      //if (accCount / replCount > 1.0) {
      //  Console.WriteLine();
      //}

      //if(accuracySum / evaluationCount > 1.0) {
      //  Console.WriteLine();
      //}

      //if (accCount < replCount && !accDiscount) accDiscount = true;
      //if (accDiscount && accuracySum/evaluationCount >= 1.0) {
      //  Console.WriteLine();
      //}

      lastStateValue = curStateValue;

      // detection of state change
      //for (int i = replayedIndex; i < Datastream.FitnessPartition.End; i++) {
      //  curStateValue = problemData.Dataset.GetDoubleValue("h", i);
      //  if (lastStateValue > curStateValue || lastStateValue < curStateValue) {
      //    stateChangeInStep = true;
      //    changeDetectionCount++;
      //  }
      //}
      //lastStateValue = curStateValue;

      // detection accuracy
      //if (!stateChangeInStep && winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
      //  // (stable) state - detected
      //  accuracySum++;
      //} else if(!stateChangeInStep && winningEnsemblePR2 < ensembles[winningEnsembleIdx].QualityThreshold.Start) {
      //  // (stable) state - not detected
      //  // no accuracy points (or penalty)
      //} else if (stateChangeInStep && winningEnsemblePR2 < ensembles[winningEnsembleIdx].QualityThreshold.Start) {
      //  // change - detected
      //  accuracySum++;
      //} else if (stateChangeInStep && winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
      //  // change - not detected
      //  // no accuracy points (or penalty)
      //}
    }
    private bool accDiscount = false;

    private double CalulateEnsembleConfidenceCM1(RatedEnsembleModel ensemble, IEnumerable<double[]> estimatedValuesPerModelPerRow, IEnumerable<double> realValues) {
      int noModels = estimatedValuesPerModelPerRow.Count();
      int votes = 0;

      OnlineCalculatorError error;

      foreach(var model in estimatedValuesPerModelPerRow) {
        var pR = OnlinePearsonsRCalculator.Calculate(model, realValues, out error);
        var pR2 = error == OnlineCalculatorError.None ? pR * pR : 0.0;
        if (pR2 >= ensemble.QualityThreshold.End) {
          votes++;
        }
      }
      

      return (double)votes/noModels;
    }

    private void timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) {
      System.Timers.Timer timer = (System.Timers.Timer)sender;
      timer.Enabled = false;
      DateTime now = DateTime.UtcNow;
      ExecutionTime += now - lastUpdateTime;
      lastUpdateTime = now;
      timer.Enabled = true;
    }

    public void CollectParameterValues(IDictionary<string, IItem> values) {
      values.Add("Datastream Analysis Name", new StringValue(Name));
      if (Datastream != null) {
        Datastream.CollectParameterValues(values);
        values.Add("Datastream Name", new StringValue(Datastream.Name));
      }
    }

    public void CollectResultValues(IDictionary<string, IItem> values) {
      values.Add("Execution Time", new TimeSpanValue(ExecutionTime));
      Results.CollectResultValues(values);
    }
    #endregion

    #region events

    #region events registration

    public EventHandler EnsemblesChanged;
    public EventHandler DatastreamChanged;

    protected virtual void DeregisterRunsEvents() {
      runs.CollectionReset -= new CollectionItemsChangedEventHandler<IRun>(Runs_CollectionReset);
    }

    protected virtual void RegisterRunsEvents() {
      runs.CollectionReset += new CollectionItemsChangedEventHandler<IRun>(Runs_CollectionReset);
    }

    protected virtual void RegisterDatastreamEvents() {
      datastream.Reset += new EventHandler(Datastream_Reset);
      datastream.ProblemDataChanged += new EventHandler(Datastream_ProblemDataChanged);
    }

    protected virtual void DeregisterDatastreamEvents() {
      datastream.Reset -= new EventHandler(Datastream_Reset);
      datastream.ProblemDataChanged -= new EventHandler(Datastream_ProblemDataChanged);
    }

    protected virtual void RegisterEnsembleEvents() {
      ensembles.ItemsAdded += new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
      ensembles.ItemsMoved += new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
      ensembles.ItemsRemoved += new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
      ensembles.ItemsReplaced += new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
      ensembles.CollectionReset += new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_Reset);
    }

    protected virtual void DeregisterEnsembleEvents() {
      ensembles.ItemsAdded -= new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
      ensembles.ItemsMoved -= new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
      ensembles.ItemsRemoved -= new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
      ensembles.ItemsReplaced -= new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
      ensembles.CollectionReset -= new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_Reset);
    }
    #endregion

    #region event handling

    protected virtual void Runs_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
      runsCounter = runs.Count;
    }

    protected virtual void Datastream_Reset(object sender, EventArgs e) {
      Prepare();
    }

    protected virtual void Datastream_ProblemDataChanged(object sender, EventArgs e) {
      Prepare();
    }

    protected virtual void Ensembles_Reset(object sender, EventArgs e) {
      Prepare();
    }

    protected virtual void Ensembles_ItemsChanged(object sender, EventArgs e) {
      Prepare();
    }

    private void OnEnsemblesChanged() {
      var changed = EnsemblesChanged;
      if (changed != null)
        changed(this, EventArgs.Empty);
    }

    private void OnDatastreamChanged() {
      var changed = DatastreamChanged;
      if (changed != null)
        changed(this, EventArgs.Empty);
    }

    #endregion

    #endregion

    #region NamedItem

    [Storable]
    protected string name;

    /// <inheritdoc/>
    /// <remarks>Calls <see cref="OnNameChanging"/> and also <see cref="OnNameChanged"/> 
    /// eventually in the setter.</remarks>
    public string Name {
      get { return name; }
      set {
        if (!CanChangeName) throw new NotSupportedException("Name cannot be changed.");
        if (!(name.Equals(value) || (value == null) && (name == string.Empty))) {
          CancelEventArgs<string> e = value == null
            ? new CancelEventArgs<string>(string.Empty)
            : new CancelEventArgs<string>(value);
          OnNameChanging(e);
          if (!e.Cancel) {
            name = value == null ? string.Empty : value;
            OnNameChanged();
          }
        }
      }
    }

    public virtual bool CanChangeName {
      get { return true; }
    }

    [Storable] protected string description;

    public string Description {
      get { return description; }
      set {
        if (!CanChangeDescription) throw new NotSupportedException("Description cannot be changed.");
        if (!(description.Equals(value) || (value == null) && (description == string.Empty))) {
          description = value == null ? string.Empty : value;
          OnDescriptionChanged();
        }
      }
    }

    public virtual bool CanChangeDescription {
      get { return true; }
    }

    /// <summary>
    /// Gets the string representation of the current instance in the format: <c>Name: [null|Value]</c>.
    /// </summary>
    /// <returns>The current instance as a string.</returns>
    public override string ToString() {
      return Name;
    }

    /// <inheritdoc/>
    public event EventHandler<CancelEventArgs<string>> NameChanging;

    /// <summary>
    /// Fires a new <c>NameChanging</c> event.
    /// </summary>
    /// <param name="e">The event arguments of the changing.</param>
    protected virtual void OnNameChanging(CancelEventArgs<string> e) {
      var handler = NameChanging;
      if (handler != null) handler(this, e);
    }

    /// <inheritdoc/>
    public event EventHandler NameChanged;

    /// <summary>
    /// Fires a new <c>NameChanged</c> event.
    /// </summary>
    /// <remarks>Calls <see cref="ItemBase.OnChanged"/>.</remarks>
    protected virtual void OnNameChanged() {
      var handler = NameChanged;
      if (handler != null) handler(this, EventArgs.Empty);
      OnToStringChanged();
    }

    /// <inheritdoc/>
    public event EventHandler DescriptionChanged;

    /// <summary>
    /// Fires a new <c>DescriptionChanged</c> event.
    /// </summary>
    /// <remarks>Calls <see cref="ItemBase.OnChanged"/>.</remarks>
    protected virtual void OnDescriptionChanged() {
      var handler = DescriptionChanged;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    #endregion nameditem
  }
}