Free cookie consent management tool by TermsFeed Policy Generator

source: branches/2719_HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4/DatastreamAnalysisOptimizer.cs @ 15867

Last change on this file since 15867 was 15867, checked in by jzenisek, 5 years ago

#2719: added proxy ensemble model

File size: 31.8 KB
Line 
1#region License Information
2
3/* HeuristicLab
4 * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
5 *
6 * This file is part of HeuristicLab.
7 *
8 * HeuristicLab is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * (at your option) any later version.
12 *
13 * HeuristicLab is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
20 */
21
22#endregion
23
24using System;
25using System.Collections.Generic;
26using System.ComponentModel;
27using System.Diagnostics;
28using System.Drawing;
29using System.Linq;
30using System.Threading;
31using System.Threading.Tasks;
32using HeuristicLab.Analysis;
33using HeuristicLab.Collections;
34using HeuristicLab.Common;
35using HeuristicLab.Core;
36using HeuristicLab.Data;
37using HeuristicLab.Optimization;
38using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
39using HeuristicLab.Problems.DataAnalysis;
40
41namespace HeuristicLab.DatastreamAnalysis {
42  internal enum DatastreamAnalysisOptimizerAction {
43    None,
44    Prepare,
45    Start,
46    Stop,
47    Pause
48  }
49
50  [StorableClass]
51  [Item("DatastreamAnalysis Optimizer",
52     "The main loop for evaluating ensemble models against a incoming datastream of time series fashion.")]
53  [Creatable(CreatableAttribute.Categories.Algorithms)]
54  public class DatastreamAnalysisOptimizer : Executable, IOptimizer, IStorableContent {
55    #region properties
56    public string Filename { get; set; }
57
58    private DatastreamAnalysisOptimizerAction daoAction;
59
60    public IEnumerable<IOptimizer> NestedOptimizers { get; }
61
62
63    [Storable]
64    protected ILog log;
65    public ILog Log {
66      get { return log; }
67    }
68
69    [Storable]
70    private ResultCollection results;
71
72    public ResultCollection Results {
73      get { return results; }
74    }
75
76    //private double lastStateValue;
77    private double pR2Sum;
78    private int detectionCount;
79    private int evaluationCount;
80    private double exclusivitySum;
81    private double accuracySum;
82
83    private CancellationTokenSource cancellationTokenSource;
84    private bool stopPending;
85    private DateTime lastUpdateTime;
86    private bool prepared;
87    private bool finished;
88
89    [Storable]
90    protected int runsCounter;
91
92    [Storable]
93    private RunCollection runs;
94
95    public RunCollection Runs
96    {
97      get { return runs; }
98      protected set
99      {
100        if (value == null) throw new ArgumentNullException();
101        if (runs != value) {
102          if (runs != null) DeregisterRunsEvents();
103          runs = value;
104          if (runs != null) RegisterRunsEvents();
105        }
106      }
107    }
108
109    [Storable]
110    private IItemList<ProxyEnsembleModel> ensembles;
111
112    public IItemList<ProxyEnsembleModel> Ensembles {
113      get { return ensembles; }
114      set {
115        if (value == null || value == ensembles)
116          return;
117        if (!(value is IProxyEnsembleModel)) throw new ArgumentException("Invaid ensemble model type");
118        DeregisterEnsembleEvents();
119        ensembles = value;
120        RegisterEnsembleEvents();
121        OnEnsemblesChanged();
122        Prepare();
123      }
124    }
125
126
127    // VAR 1: datastream ~= problem data, VAR 2 (TODO): datastream = external source e.g. webservice, AMQP-Queue, etc.
128    [Storable]
129    private Datastream datastream;
130
131    public Datastream Datastream {
132      get { return datastream; }
133      set {
134        if (value == null || value == datastream)
135          return;
136        if(!(value is IDatastream)) throw new ArgumentException("Invalid datastream type");
137        DeregisterDatastreamEvents();
138        datastream = value;
139        RegisterDatastreamEvents();
140        OnDatastreamChanged();
141        Prepare();
142      }
143    }
144
145    #endregion properties
146
147    #region results properties
148    private int ResultsSlidingWindowMovements {
149      get { return ((IntValue)Results["Sliding Window Movements"].Value).Value; }
150      set { ((IntValue)Results["Sliding Window Movements"].Value).Value = value; }
151    }
152    private DataTable ResultsQualities {
153      get { return ((DataTable)Results["Qualities 1"].Value); }
154    }
155
156    private DataTable ResultsPeakQualities {
157      get { return ((DataTable) Results["Qualities 2"].Value); }
158    }
159
160    private const string ResultsQualitiesMSE = "Mean squared error";
161    private const string ResultsQualitiesPR2 = "Pearson R²";
162
163    private DataTable ResultsTargets {
164      get { return ((DataTable) Results["Targets"].Value); }
165    }
166
167    private DataTable ResultsStateDetection {
168      get { return ((DataTable) Results["State Detection"].Value);}
169    }
170
171    private const string ResultsTargetsReal = "Real";
172
173    private DataBarSet ResultsSWQualitiesBars {
174      get { return (DataBarSet) Results["Ensemble Comparison [SW-Quality]"].Value; }
175    }
176
177    private DataBarSet ResultsSWVotingBars {
178      get { return (DataBarSet) Results["Ensemble Comparison [SW-Voting]"].Value; }
179    }
180
181    private DataBarSet ResultsSonarQualitiesBars {
182      get { return (DataBarSet)Results["Ensemble Comparison [Sonar-Quality]"].Value; }
183    }
184
185    private DataBarSet ResultsSWPeakQualitiesBars {
186      get { return (DataBarSet)Results["Ensemble Comparison [SW-PeakQuality]"].Value; }
187    }
188
189    private double ResultsMeanPR2 {
190      get { return ((DoubleValue) Results["Mean PR2"].Value).Value; }
191      set { ((DoubleValue)Results["Mean PR2"].Value).Value = value; }
192    }
193
194    private double ResultsDetectionAccuracy {
195      get { return ((DoubleValue)Results["Detection Accuracy"].Value).Value; }
196      set { ((DoubleValue)Results["Detection Accuracy"].Value).Value = value; }
197    }
198
199    private double ResultsDetectionExclusivity {
200      get { return ((DoubleValue)Results["Detection Exclusivity"].Value).Value; }
201      set { ((DoubleValue)Results["Detection Exclusivity"].Value).Value = value; }
202    }
203
204    //private int ResultsTrueDetectionCount {
205    //  get { return ((IntValue)Results["True Detection Count"].Value).Value; }
206    //  set { ((IntValue)Results["True Detection Count"].Value).Value = value; }
207    //}
208
209    //private int ResultsFalseDetectionCount {
210    //  get { return ((IntValue)Results["False Detection Count"].Value).Value; }
211    //  set { ((IntValue)Results["False Detection Count"].Value).Value = value; }
212    //}
213
214    protected void SetupResults() {
215      evaluationCount = 0;
216      //lastStateValue = 0.0;
217      pR2Sum = 0.0;
218      exclusivitySum = 0.0;
219      accuracySum = 0.0;
220      detectionCount = 0;
221      Results.Clear();
222      Results.Add(new Result("Sliding Window Movements", new IntValue(0)));
223      Results.Add(new Result("Qualities 1", new DataTable("Average Pearson R²")));
224      Results.Add(new Result("Qualities 2", new DataTable("Peak Pearson R²")));
225      Results.Add(new Result("Targets", new DataTable("Targets")));
226      Results.Add(new Result("State Detection", new DataTable("State Detection")));
227      Results.Add(new Result("Ensemble Comparison [SW-Quality]", new DataBarSet("Ensemble Comparison [SW-Quality]")));
228      Results.Add(new Result("Ensemble Comparison [SW-Voting]", new DataBarSet("Ensemble Comparison [SW-Voting]")));
229      Results.Add(new Result("Ensemble Comparison [Sonar-Quality]", new DataBarSet("Ensemble Comparison [Sonar-Quality]")));
230      Results.Add(new Result("Ensemble Comparison [SW-PeakQuality]", new DataBarSet("Ensemble Comparison [SW-PeakQuality]")));
231      Results.Add(new Result("Mean PR2", new DoubleValue()));
232      Results.Add(new Result("Detection Accuracy", new DoubleValue()));
233      Results.Add(new Result("Detection Exclusivity", new DoubleValue()));
234
235      ResultsTargets.Rows.Add(new DataRow(ResultsTargetsReal));
236      foreach (var ensemble in Ensembles) {
237        // targets table
238        ResultsTargets.Rows.Add(new DataRow(ensemble.Name));
239
240        ResultsStateDetection.Rows.Add(new DataRow(ensemble.Name));
241
242        // qualities (series)
243        //ResultsQualities.Rows.Add(new DataRow(ensemble.Name + " - " + ResultsQualitiesMSE));
244        ResultsQualities.Rows.Add(new DataRow(ensemble.Name));
245
246        ResultsPeakQualities.Rows.Add(new DataRow(ensemble.Name));
247
248        // qualities (bars)
249        ResultsSWQualitiesBars.Bars.Add(new DataBar(ensemble.Name, ensemble.QualityThreshold.Start, ensemble.QualityThreshold.End));
250
251        // voting (bars)
252        ResultsSWVotingBars.Bars.Add(new DataBar(ensemble.Name, ensemble.ConfidenceThreshold.Start, ensemble.ConfidenceThreshold.End));
253
254        // sonar quality (bars)
255        ResultsSonarQualitiesBars.Bars.Add(new DataBar(ensemble.Name, ensemble.QualityThreshold.Start, ensemble.QualityThreshold.End));
256
257        // quality peaks (bars)
258        ResultsSWPeakQualitiesBars.Bars.Add(new DataBar(ensemble.Name, ensemble.QualityThreshold.Start, ensemble.QualityThreshold.End));
259      }
260    }
261
262    #endregion
263
264    #region constructors, cloner,...
265    public DatastreamAnalysisOptimizer() : base() {
266      name = "Datastream Analysis";
267      log = new Log();
268      results = new ResultCollection();
269      ensembles = new ItemList<ProxyEnsembleModel>();
270      datastream = new Datastream();
271      runsCounter = 0;
272      runs = new RunCollection();
273      Initialize();
274    }
275
276    [StorableConstructor]
277    protected DatastreamAnalysisOptimizer(bool deserializing) : base(deserializing) {
278    }
279
280    [StorableHook(HookType.AfterDeserialization)]
281    private void AfterDeserialization() {
282      Initialize();
283    }
284
285    protected DatastreamAnalysisOptimizer(DatastreamAnalysisOptimizer original, Cloner cloner) : base(original, cloner) {
286      name = original.name;
287      log = cloner.Clone(original.log);
288      results = cloner.Clone(original.results);
289      ensembles = (ItemList<ProxyEnsembleModel>) original.Ensembles.Clone(cloner);
290      datastream = (Datastream) original.Datastream.Clone(cloner);
291      runsCounter = original.runsCounter;
292      runs = cloner.Clone(original.runs);
293      Initialize();
294    }
295
296    public override IDeepCloneable Clone(Cloner cloner) {
297      return new DatastreamAnalysisOptimizer(this, cloner);
298    }
299
300    private void Initialize() {
301      if (runs != null) RegisterRunsEvents();
302      if (datastream != null) RegisterDatastreamEvents();
303      if (ensembles != null) RegisterEnsembleEvents();
304    }
305    #endregion
306
307    #region control actions
308
309    public override void Prepare() {
310      if (ensembles == null || ensembles.Count == 0 || datastream == null || !datastream.SlidingWindowEvaluationPossible) return;
311      //if (ensembles.SelectMany(x => x.Models).Count() == 0) return;
312      base.Prepare();
313      OnPrepared();
314    }
315
316    public void Prepare(bool clearRuns) {
317      if (ensembles == null || ensembles.Count == 0 || datastream == null || !datastream.SlidingWindowEvaluationPossible) return;
318
319      base.Prepare();
320      if (clearRuns) runs.Clear();
321      OnPrepared();
322    }
323
324    public override void Start(CancellationToken cancellationToken) {
325      base.Start(cancellationToken);
326      if (ensembles == null || datastream == null) return;
327
328      cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
329      stopPending = false;
330
331      if (prepared) {
332        SetupResults();
333        Datastream.InitializeState();
334      }
335
336      Task task = Task.Factory.StartNew(Run, cancellationTokenSource.Token, cancellationTokenSource.Token);
337      task.ContinueWith(t => {
338        try {
339          t.Wait();
340        }
341        catch (AggregateException ex) {
342          try {
343            ex.Flatten().Handle(x => x is OperationCanceledException);
344          } catch (AggregateException remaining) {
345            if(remaining.InnerExceptions.Count == 1) OnExceptionOccurred(remaining.InnerExceptions[0]);
346            else OnExceptionOccurred(remaining);
347          }
348        }
349        cancellationTokenSource.Dispose();
350        cancellationTokenSource = null;
351
352        // handle stop/pause
353        if (stopPending || finished) {
354          OnStopped();
355        } else {
356          OnPaused();
357        }
358      });
359    }
360
361    public override void Pause() {
362      if (ensembles == null || datastream == null) return;
363      base.Pause();
364      cancellationTokenSource.Cancel();
365    }
366
367    public override void Stop() {
368      if (ensembles == null || datastream == null) return;
369      base.Stop();
370      if (ExecutionState == ExecutionState.Paused) {
371        OnStopped();
372      } else {
373        stopPending = true;
374        cancellationTokenSource.Cancel();
375      }
376    }
377
378    protected override void OnPrepared() {
379      ExecutionTime = TimeSpan.Zero;
380      foreach (IStatefulItem statefulItem in this.GetObjectGraphObjects(new HashSet<object>() {Runs}).OfType<IStatefulItem>()) {
381        statefulItem.InitializeState();
382      }
383      results.Clear();
384      prepared = true;
385      finished = false;
386      Log.LogMessage("Datastream analysis prepared");
387      base.OnPrepared();
388    }
389
390    protected override void OnStarted() {
391      Log.LogMessage("Datastream analysis started");
392      base.OnStarted();
393    }
394
395    protected override void OnPaused() {
396      Log.LogMessage("Datastream analysis paused");
397      base.OnPaused();
398    }
399
400    protected override void OnStopped() {
401      try {
402        runsCounter++;
403        var run = new Run();
404        run.Filename = Filename;
405        run.Name = string.Format("{0} Run {1}", Name, runsCounter);
406        CollectParameterValues(run.Parameters);
407        CollectResultValues(run.Results);
408        runs.Add(run);
409      }
410      finally {
411        Log.LogMessage("Datastream analysis stopped");
412        base.OnStopped();
413      }
414    }
415
416    #endregion
417
418    #region run
419    private void Run(object state) {
420      CancellationToken cancellationToken = (CancellationToken) state;
421      OnStarted();
422      lastUpdateTime = DateTime.UtcNow;
423      System.Timers.Timer timer = new System.Timers.Timer(250);
424      timer.AutoReset = true;
425      timer.Elapsed += new System.Timers.ElapsedEventHandler(timer_Elapsed);
426      timer.Start();
427
428      try {
429        Run(cancellationToken);
430      }
431      finally {
432        timer.Elapsed -= new System.Timers.ElapsedEventHandler(timer_Elapsed);
433        timer.Stop();
434        ExecutionTime += DateTime.UtcNow - lastUpdateTime;
435      }
436
437      cancellationToken.ThrowIfCancellationRequested();
438    }
439
440    private int replayedIndex;
441
442    protected void Run(CancellationToken cancellationToken) {
443
444      if (prepared) {
445        replayedIndex = 0;
446        prepared = false;
447      }
448
449      try {
450
451        // play and evaluate initial window
452        PlayDatastream();
453        //lastStateValue = Datastream.ProblemData.Dataset.GetDoubleValue("h", replayedIndex);
454        if (Datastream.SlidingWindowEvaluationPossible) Evaluate();
455        replayedIndex = Datastream.FitnessPartition.End;
456
457        do {
458          while(Datastream.SlidingWindowMovementPossible) {
459            cancellationToken.ThrowIfCancellationRequested();
460
461            // perform (delayed) window movement
462            Thread.Sleep(Datastream.SlidingWindowMovementDelay.Value);
463            Datastream.MoveSlidingWindow();
464            ResultsSlidingWindowMovements++;
465
466            // play and evaluate the moved window
467            PlayDatastream();
468            if (Datastream.SlidingWindowEvaluationPossible) Evaluate();
469            replayedIndex = Datastream.FitnessPartition.End;
470          }
471        } while (Datastream.UpdateAvailable);
472        finished = true;
473      }
474      catch (Exception ex) {
475        if (ex is ArgumentOutOfRangeException) throw ex;
476        if (ex is OperationCanceledException) throw ex;
477      }
478      finally {
479        // reset everything
480        //Prepare(true);
481      }
482    }
483
484    private void PlayDatastream() {
485      var problemData = Datastream.ProblemData;
486      var targetVarName = problemData.TargetVariable;
487
488      for (int i = replayedIndex; i < Datastream.FitnessPartition.End; i++) {
489        var realValue = problemData.Dataset.GetDoubleValue(targetVarName, i);
490        ResultsTargets.Rows[ResultsTargetsReal].Values.Add(realValue);
491      }
492    }
493
494    private void Smoothen(double[] estimatedValuesPerRow) {
495      if (Datastream.SlidingWindowEvaluationScheme.Value == Datastream.EvaluationScheme.equal) {
496        var avg = estimatedValuesPerRow.Average();
497        for (int i = 0; i < estimatedValuesPerRow.Length; i++) {
498          estimatedValuesPerRow[i] = avg;
499        }
500      }
501    }
502
503    private void Evaluate() {
504      evaluationCount++;
505      var problemData = Datastream.ProblemData;
506      var dataset = problemData.Dataset;
507      var targetVarName = problemData.TargetVariable;
508
509      var rows = Enumerable.Range(Datastream.FitnessPartition.Start, Datastream.FitnessPartition.Size);
510      var realValues = dataset.GetDoubleValues(targetVarName, rows);
511
512      int sonarSize = (int)(Datastream.FitnessPartition.Size * Datastream.SlidingWindowSonarRatio.Value);
513      var sonarStart = Datastream.FitnessPartition.End - sonarSize;
514      var sonarRows = Enumerable.Range(sonarStart, sonarSize);
515      var sonarRealValues = dataset.GetDoubleValues(targetVarName, sonarRows);
516
517      var winningEnsembleName = ensembles[0].Name;
518      var winningEnsemblePR2 = 0.0;
519      var winningEnsembleIdx = 0;
520      var ec = 0;
521      bool stateChangeInStep = false;
522      var curDetectionCount = 0;
523      var changeDetectionCount = 0;
524      //double curStateValue = 0.0;
525
526      foreach (var ensemble in Ensembles) {
527
528        var sonarEstimatedValuesPerModelPerRow = ensemble.Model.Models.Select(x => x.GetEstimatedValues(datastream.ProblemData.Dataset, sonarRows).ToArray());
529        var sonarEstimatedValuesPerRow = Enumerable.Range(0, sonarSize).Select(r => sonarEstimatedValuesPerModelPerRow.Select(m => m[r]).Average()).ToArray();
530       
531        var estimatedValuesPerModelPerRow = ensemble.Model.Models.Select(x => x.GetEstimatedValues(datastream.ProblemData.Dataset, rows).ToArray());
532        var estimatedValuesPerRow = Enumerable.Range(0, Datastream.FitnessPartition.Size).Select(r => estimatedValuesPerModelPerRow.Select(e => e[r]).Average()).ToArray(); // per row
533        var averageEstimatedValuesPerModel = estimatedValuesPerModelPerRow.Select(x => x.Average()); // per model
534        var averageEstimatedValue = averageEstimatedValuesPerModel.Average();
535
536        // calulate quality
537        //var mse = Math.Pow(averageEstimatedValue - realValues.Average(), 2);
538        OnlineCalculatorError error;
539        var pR = OnlinePearsonsRCalculator.Calculate(estimatedValuesPerRow, realValues, out error);
540        var pR2 = error == OnlineCalculatorError.None ? pR * pR : 0.0;
541
542        var sonarPR = OnlinePearsonsRCalculator.Calculate(sonarEstimatedValuesPerRow, sonarRealValues, out error);
543        var sonarPR2 = error == OnlineCalculatorError.None ? sonarPR * sonarPR : 0.0;
544   
545        var estimatedValuesPerModelPerRowArr = estimatedValuesPerModelPerRow.ToArray();
546        var curPR = 0.0;
547       
548        var peakPR = OnlinePearsonsRCalculator.Calculate(estimatedValuesPerModelPerRowArr[0], realValues, out error);
549        for (int i = 1; i < estimatedValuesPerModelPerRow.Count(); i++) {
550          curPR = OnlinePearsonsRCalculator.Calculate(estimatedValuesPerModelPerRowArr[i], realValues, out error);
551          if (curPR > peakPR && error == OnlineCalculatorError.None) {
552            peakPR = curPR;
553          }
554        }
555        var peakPR2 = peakPR * peakPR;
556
557        // calculate confidence
558        var cm1 = CalulateEnsembleConfidenceCM1(ensemble, estimatedValuesPerModelPerRow, realValues);
559
560        int replayAmount = Datastream.FitnessPartition.End - replayedIndex;
561        int replayCount = estimatedValuesPerRow.Length - replayAmount;
562
563        Smoothen(estimatedValuesPerRow);
564
565        for (int i = replayedIndex; i < Datastream.FitnessPartition.End; i++) {
566          //ResultsTargets.Rows[ensemble.Name].Values.Add(averageEstimatedValue);
567          ResultsTargets.Rows[ensemble.Name].Values.Add(estimatedValuesPerRow[replayCount]);
568          replayCount++;
569
570          //ResultsQualities.Rows[ensemble.Name + " - " + ResultsQualitiesMSE].Values.Add(mse);
571          ResultsQualities.Rows[ensemble.Name].Values.Add(pR2);
572          ResultsPeakQualities.Rows[ensemble.Name].Values.Add(peakPR2);
573
574          double amp = (pR2 > ensemble.QualityThreshold.Start) ? 1.0 : 0.0;
575          ResultsStateDetection.Rows[ensemble.Name].Values.Add(amp);   
576        }
577        ResultsSWQualitiesBars.Bars[ensemble.Name].Value = pR2;
578        ResultsSonarQualitiesBars.Bars[ensemble.Name].Value = sonarPR2;
579        ResultsSWVotingBars.Bars[ensemble.Name].Value = cm1;
580        ResultsSWPeakQualitiesBars.Bars[ensemble.Name].Value = peakPR2;
581
582        if (pR2 > ensemble.QualityThreshold.Start)
583          curDetectionCount++;
584
585        if (pR2 > winningEnsemblePR2) {
586          winningEnsemblePR2 = pR2;
587          winningEnsembleName = ensemble.Name;
588          winningEnsembleIdx = ec;
589        }
590        ec++;
591      }
592
593      // quality meassures
594      // =================================================
595
596      if (winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
597        // Mean PR2
598        pR2Sum += winningEnsemblePR2;
599        detectionCount++;
600        ResultsMeanPR2 = pR2Sum/detectionCount;
601
602        // detection exclusivity --> 1.0 - (x - 1) * (1.0 / n)
603        var fraction = 1.0 / ensembles.Count;
604        var curExclusivity = 1.0 - (curDetectionCount - 1) * fraction;
605        exclusivitySum += curExclusivity;
606        ResultsDetectionExclusivity = exclusivitySum / detectionCount;
607      }
608
609      //double replCount = 0.0;
610      //double accCount = 0.0;
611      //for (int i = replayedIndex; i < Datastream.FitnessPartition.End; i++) {
612      //  replCount++;
613      //  //curStateValue = problemData.Dataset.GetDoubleValue("h", i);
614      //  if (curStateValue <= 0.5 && winningEnsembleIdx == 0 && winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
615      //    accCount++;
616      //  } else if (curStateValue > 0.5 && winningEnsembleIdx == 1 && winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
617      //    accCount++;
618      //  }
619      //}
620      //accuracySum += accCount / replCount;
621      //ResultsDetectionAccuracy = accuracySum / evaluationCount;
622
623
624
625
626      //if (accCount / replCount > 1.0) {
627      //  Console.WriteLine();
628      //}
629
630      //if(accuracySum / evaluationCount > 1.0) {
631      //  Console.WriteLine();
632      //}
633
634      //if (accCount < replCount && !accDiscount) accDiscount = true;
635      //if (accDiscount && accuracySum/evaluationCount >= 1.0) {
636      //  Console.WriteLine();
637      //}
638
639      //lastStateValue = curStateValue;
640
641      // detection of state change
642      //for (int i = replayedIndex; i < Datastream.FitnessPartition.End; i++) {
643      //  curStateValue = problemData.Dataset.GetDoubleValue("h", i);
644      //  if (lastStateValue > curStateValue || lastStateValue < curStateValue) {
645      //    stateChangeInStep = true;
646      //    changeDetectionCount++;
647      //  }
648      //}
649      //lastStateValue = curStateValue;
650
651      // detection accuracy
652      //if (!stateChangeInStep && winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
653      //  // (stable) state - detected
654      //  accuracySum++;
655      //} else if(!stateChangeInStep && winningEnsemblePR2 < ensembles[winningEnsembleIdx].QualityThreshold.Start) {
656      //  // (stable) state - not detected
657      //  // no accuracy points (or penalty)
658      //} else if (stateChangeInStep && winningEnsemblePR2 < ensembles[winningEnsembleIdx].QualityThreshold.Start) {
659      //  // change - detected
660      //  accuracySum++;
661      //} else if (stateChangeInStep && winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
662      //  // change - not detected
663      //  // no accuracy points (or penalty)
664      //}
665    }
666    private bool accDiscount = false;
667
668    private double CalulateEnsembleConfidenceCM1(ProxyEnsembleModel ensemble, IEnumerable<double[]> estimatedValuesPerModelPerRow, IEnumerable<double> realValues) {
669      int noModels = estimatedValuesPerModelPerRow.Count();
670      int votes = 0;
671
672      OnlineCalculatorError error;
673
674      foreach(var model in estimatedValuesPerModelPerRow) {
675        var pR = OnlinePearsonsRCalculator.Calculate(model, realValues, out error);
676        var pR2 = error == OnlineCalculatorError.None ? pR * pR : 0.0;
677        if (pR2 >= ensemble.QualityThreshold.End) {
678          votes++;
679        }
680      }
681     
682
683      return (double)votes/noModels;
684    }
685
686    private void timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) {
687      System.Timers.Timer timer = (System.Timers.Timer)sender;
688      timer.Enabled = false;
689      DateTime now = DateTime.UtcNow;
690      ExecutionTime += now - lastUpdateTime;
691      lastUpdateTime = now;
692      timer.Enabled = true;
693    }
694
695    public void CollectParameterValues(IDictionary<string, IItem> values) {
696      values.Add("Datastream Analysis Name", new StringValue(Name));
697      if (Datastream != null) {
698        Datastream.CollectParameterValues(values);
699        values.Add("Datastream Name", new StringValue(Datastream.Name));
700      }
701    }
702
703    public void CollectResultValues(IDictionary<string, IItem> values) {
704      values.Add("Execution Time", new TimeSpanValue(ExecutionTime));
705      Results.CollectResultValues(values);
706    }
707    #endregion
708
709    #region events
710
711    #region events registration
712
713    public EventHandler EnsemblesChanged;
714    public EventHandler DatastreamChanged;
715
716    protected virtual void DeregisterRunsEvents() {
717      runs.CollectionReset -= new CollectionItemsChangedEventHandler<IRun>(Runs_CollectionReset);
718    }
719
720    protected virtual void RegisterRunsEvents() {
721      runs.CollectionReset += new CollectionItemsChangedEventHandler<IRun>(Runs_CollectionReset);
722    }
723
724    protected virtual void RegisterDatastreamEvents() {
725      datastream.Reset += new EventHandler(Datastream_Reset);
726      datastream.ProblemDataChanged += new EventHandler(Datastream_ProblemDataChanged);
727    }
728
729    protected virtual void DeregisterDatastreamEvents() {
730      datastream.Reset -= new EventHandler(Datastream_Reset);
731      datastream.ProblemDataChanged -= new EventHandler(Datastream_ProblemDataChanged);
732    }
733
734    protected virtual void RegisterEnsembleEvents() {
735      ensembles.ItemsAdded += new CollectionItemsChangedEventHandler<IndexedItem<ProxyEnsembleModel>>(Ensembles_ItemsChanged);
736      ensembles.ItemsMoved += new CollectionItemsChangedEventHandler<IndexedItem<ProxyEnsembleModel>>(Ensembles_ItemsChanged);
737      ensembles.ItemsRemoved += new CollectionItemsChangedEventHandler<IndexedItem<ProxyEnsembleModel>>(Ensembles_ItemsChanged);
738      ensembles.ItemsReplaced += new CollectionItemsChangedEventHandler<IndexedItem<ProxyEnsembleModel>>(Ensembles_ItemsChanged);
739      ensembles.CollectionReset += new CollectionItemsChangedEventHandler<IndexedItem<ProxyEnsembleModel>>(Ensembles_Reset);
740    }
741
742    protected virtual void DeregisterEnsembleEvents() {
743      ensembles.ItemsAdded -= new CollectionItemsChangedEventHandler<IndexedItem<ProxyEnsembleModel>>(Ensembles_ItemsChanged);
744      ensembles.ItemsMoved -= new CollectionItemsChangedEventHandler<IndexedItem<ProxyEnsembleModel>>(Ensembles_ItemsChanged);
745      ensembles.ItemsRemoved -= new CollectionItemsChangedEventHandler<IndexedItem<ProxyEnsembleModel>>(Ensembles_ItemsChanged);
746      ensembles.ItemsReplaced -= new CollectionItemsChangedEventHandler<IndexedItem<ProxyEnsembleModel>>(Ensembles_ItemsChanged);
747      ensembles.CollectionReset -= new CollectionItemsChangedEventHandler<IndexedItem<ProxyEnsembleModel>>(Ensembles_Reset);
748    }
749    #endregion
750
751    #region event handling
752
753    protected virtual void Runs_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
754      runsCounter = runs.Count;
755    }
756
757    protected virtual void Datastream_Reset(object sender, EventArgs e) {
758      Prepare();
759    }
760
761    protected virtual void Datastream_ProblemDataChanged(object sender, EventArgs e) {
762      Prepare();
763    }
764
765    protected virtual void Ensembles_Reset(object sender, EventArgs e) {
766      Prepare();
767    }
768
769    protected virtual void Ensembles_ItemsChanged(object sender, EventArgs e) {
770      Prepare();
771    }
772
773    private void OnEnsemblesChanged() {
774      var changed = EnsemblesChanged;
775      if (changed != null)
776        changed(this, EventArgs.Empty);
777    }
778
779    private void OnDatastreamChanged() {
780      var changed = DatastreamChanged;
781      if (changed != null)
782        changed(this, EventArgs.Empty);
783    }
784
785    #endregion
786
787    #endregion
788
789    #region NamedItem
790
791    [Storable]
792    protected string name;
793
794    /// <inheritdoc/>
795    /// <remarks>Calls <see cref="OnNameChanging"/> and also <see cref="OnNameChanged"/>
796    /// eventually in the setter.</remarks>
797    public string Name {
798      get { return name; }
799      set {
800        if (!CanChangeName) throw new NotSupportedException("Name cannot be changed.");
801        if (!(name.Equals(value) || (value == null) && (name == string.Empty))) {
802          CancelEventArgs<string> e = value == null
803            ? new CancelEventArgs<string>(string.Empty)
804            : new CancelEventArgs<string>(value);
805          OnNameChanging(e);
806          if (!e.Cancel) {
807            name = value == null ? string.Empty : value;
808            OnNameChanged();
809          }
810        }
811      }
812    }
813
814    public virtual bool CanChangeName {
815      get { return true; }
816    }
817
818    [Storable] protected string description;
819
820    public string Description {
821      get { return description; }
822      set {
823        if (!CanChangeDescription) throw new NotSupportedException("Description cannot be changed.");
824        if (!(description.Equals(value) || (value == null) && (description == string.Empty))) {
825          description = value == null ? string.Empty : value;
826          OnDescriptionChanged();
827        }
828      }
829    }
830
831    public virtual bool CanChangeDescription {
832      get { return true; }
833    }
834
835    /// <summary>
836    /// Gets the string representation of the current instance in the format: <c>Name: [null|Value]</c>.
837    /// </summary>
838    /// <returns>The current instance as a string.</returns>
839    public override string ToString() {
840      return Name;
841    }
842
843    /// <inheritdoc/>
844    public event EventHandler<CancelEventArgs<string>> NameChanging;
845
846    /// <summary>
847    /// Fires a new <c>NameChanging</c> event.
848    /// </summary>
849    /// <param name="e">The event arguments of the changing.</param>
850    protected virtual void OnNameChanging(CancelEventArgs<string> e) {
851      var handler = NameChanging;
852      if (handler != null) handler(this, e);
853    }
854
855    /// <inheritdoc/>
856    public event EventHandler NameChanged;
857
858    /// <summary>
859    /// Fires a new <c>NameChanged</c> event.
860    /// </summary>
861    /// <remarks>Calls <see cref="ItemBase.OnChanged"/>.</remarks>
862    protected virtual void OnNameChanged() {
863      var handler = NameChanged;
864      if (handler != null) handler(this, EventArgs.Empty);
865      OnToStringChanged();
866    }
867
868    /// <inheritdoc/>
869    public event EventHandler DescriptionChanged;
870
871    /// <summary>
872    /// Fires a new <c>DescriptionChanged</c> event.
873    /// </summary>
874    /// <remarks>Calls <see cref="ItemBase.OnChanged"/>.</remarks>
875    protected virtual void OnDescriptionChanged() {
876      var handler = DescriptionChanged;
877      if (handler != null) handler(this, EventArgs.Empty);
878    }
879
880    #endregion nameditem
881  }
882}
Note: See TracBrowser for help on using the repository browser.