source: branches/Async/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs @ 15216

Last change on this file since 15216 was 15216, checked in by jkarder, 5 years ago

#2258: reset startPending if an exception is thrown before OnStarted is called

File size: 33.0 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Drawing;
25using System.Linq;
26using System.Threading;
27using System.Threading.Tasks;
28using HeuristicLab.Collections;
29using HeuristicLab.Common;
30using HeuristicLab.Core;
31using HeuristicLab.Data;
32using HeuristicLab.Optimization;
33using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
34using HeuristicLab.Problems.DataAnalysis;
35using HeuristicLab.Problems.DataAnalysis.Symbolic;
36
37namespace HeuristicLab.Algorithms.DataAnalysis {
38  [Item("Cross Validation (CV)", "Cross-validation wrapper for data analysis algorithms.")]
39  [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 100)]
40  [StorableClass]
41  public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent {
42    private SemaphoreSlim availableWorkers; // limits the number of concurrent algorithm executions
43    private ManualResetEventSlim allAlgorithmsFinished; // this indicates that all started algorithms have been paused or stopped
44
45    public CrossValidation()
46      : base() {
47      name = ItemName;
48      description = ItemDescription;
49
50      executionState = ExecutionState.Stopped;
51      runs = new RunCollection { OptimizerName = name };
52      runsCounter = 0;
53
54      algorithm = null;
55      clonedAlgorithms = new ItemCollection<IAlgorithm>();
56      results = new ResultCollection();
57
58      folds = new IntValue(2);
59      numberOfWorkers = new IntValue(1);
60      samplesStart = new IntValue(0);
61      samplesEnd = new IntValue(0);
62      storeAlgorithmInEachRun = false;
63
64      RegisterEvents();
65      if (Algorithm != null) RegisterAlgorithmEvents();
66    }
67
68    public string Filename { get; set; }
69
70    #region persistence and cloning
71    [StorableConstructor]
72    private CrossValidation(bool deserializing)
73      : base(deserializing) {
74    }
75    [StorableHook(HookType.AfterDeserialization)]
76    private void AfterDeserialization() {
77      RegisterEvents();
78      if (Algorithm != null) RegisterAlgorithmEvents();
79    }
80
81    private CrossValidation(CrossValidation original, Cloner cloner)
82      : base(original, cloner) {
83      executionState = original.executionState;
84      storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
85      runs = cloner.Clone(original.runs);
86      runsCounter = original.runsCounter;
87      algorithm = cloner.Clone(original.algorithm);
88      clonedAlgorithms = cloner.Clone(original.clonedAlgorithms);
89      results = cloner.Clone(original.results);
90
91      folds = cloner.Clone(original.folds);
92      numberOfWorkers = cloner.Clone(original.numberOfWorkers);
93      samplesStart = cloner.Clone(original.samplesStart);
94      samplesEnd = cloner.Clone(original.samplesEnd);
95      RegisterEvents();
96      if (Algorithm != null) RegisterAlgorithmEvents();
97    }
98    public override IDeepCloneable Clone(Cloner cloner) {
99      return new CrossValidation(this, cloner);
100    }
101
102    #endregion
103
104    #region properties
105    [Storable]
106    private IAlgorithm algorithm;
107    public IAlgorithm Algorithm {
108      get { return algorithm; }
109      set {
110        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
111          throw new InvalidOperationException("Changing the algorithm is only allowed if the CrossValidation is stopped or prepared.");
112        if (algorithm != value) {
113          if (value != null && value.Problem != null && !(value.Problem is IDataAnalysisProblem))
114            throw new ArgumentException("Only algorithms with a DataAnalysisProblem could be used for the cross validation.");
115          if (algorithm != null) DeregisterAlgorithmEvents();
116          algorithm = value;
117          Parameters.Clear();
118
119          if (algorithm != null) {
120            algorithm.StoreAlgorithmInEachRun = false;
121            RegisterAlgorithmEvents();
122            algorithm.Prepare(true);
123            Parameters.AddRange(algorithm.Parameters);
124          }
125          OnAlgorithmChanged();
126          Prepare();
127        }
128      }
129    }
130
131
132    [Storable]
133    private IDataAnalysisProblem problem;
134    public IDataAnalysisProblem Problem {
135      get {
136        if (algorithm == null)
137          return null;
138        return (IDataAnalysisProblem)algorithm.Problem;
139      }
140      set {
141        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
142          throw new InvalidOperationException("Changing the problem is only allowed if the CrossValidation is stopped or prepared.");
143        if (algorithm == null) throw new ArgumentNullException("Could not set a problem before an algorithm was set.");
144        algorithm.Problem = value;
145        problem = value;
146      }
147    }
148
149    IProblem IAlgorithm.Problem {
150      get { return Problem; }
151      set {
152        if (value != null && !ProblemType.IsInstanceOfType(value))
153          throw new ArgumentException("Only DataAnalysisProblems could be used for the cross validation.");
154        Problem = (IDataAnalysisProblem)value;
155      }
156    }
157    public Type ProblemType {
158      get { return typeof(IDataAnalysisProblem); }
159    }
160
161    [Storable]
162    private ItemCollection<IAlgorithm> clonedAlgorithms;
163
164    public IEnumerable<IOptimizer> NestedOptimizers {
165      get {
166        if (Algorithm == null) yield break;
167        yield return Algorithm;
168      }
169    }
170
171    [Storable]
172    private ResultCollection results;
173    public ResultCollection Results {
174      get { return results; }
175    }
176
177    [Storable]
178    private IntValue folds;
179    public IntValue Folds {
180      get { return folds; }
181    }
182    [Storable]
183    private IntValue samplesStart;
184    public IntValue SamplesStart {
185      get { return samplesStart; }
186    }
187    [Storable]
188    private IntValue samplesEnd;
189    public IntValue SamplesEnd {
190      get { return samplesEnd; }
191    }
192    [Storable]
193    private IntValue numberOfWorkers;
194    public IntValue NumberOfWorkers {
195      get { return numberOfWorkers; }
196    }
197
198    [Storable]
199    private bool storeAlgorithmInEachRun;
200    public bool StoreAlgorithmInEachRun {
201      get { return storeAlgorithmInEachRun; }
202      set {
203        if (storeAlgorithmInEachRun != value) {
204          storeAlgorithmInEachRun = value;
205          OnStoreAlgorithmInEachRunChanged();
206        }
207      }
208    }
209
210    [Storable]
211    private int runsCounter;
212    [Storable]
213    private RunCollection runs;
214    public RunCollection Runs {
215      get { return runs; }
216    }
217
218    [Storable]
219    private ExecutionState executionState;
220    public ExecutionState ExecutionState {
221      get { return executionState; }
222      private set {
223        if (executionState != value) {
224          executionState = value;
225          OnExecutionStateChanged();
226          OnItemImageChanged();
227        }
228      }
229    }
230    public static new Image StaticItemImage {
231      get { return HeuristicLab.Common.Resources.VSImageLibrary.Event; }
232    }
233    public override Image ItemImage {
234      get {
235        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
236        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
237        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
238        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
239        else return base.ItemImage;
240      }
241    }
242
243    public TimeSpan ExecutionTime {
244      get {
245        if (ExecutionState != ExecutionState.Prepared)
246          return TimeSpan.FromMilliseconds(clonedAlgorithms.Select(x => x.ExecutionTime.TotalMilliseconds).Sum());
247        return TimeSpan.Zero;
248      }
249    }
250    #endregion
251
252    protected override void OnNameChanged() {
253      base.OnNameChanged();
254      Runs.OptimizerName = Name;
255    }
256
257    public void Prepare() {
258      if (startPending) return;
259      if (ExecutionState == ExecutionState.Started)
260        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
261      results.Clear();
262      clonedAlgorithms.Clear();
263      if (Algorithm != null) {
264        Algorithm.Prepare();
265        if (Algorithm.ExecutionState == ExecutionState.Prepared) OnPrepared();
266      }
267    }
268    public void Prepare(bool clearRuns) {
269      if (clearRuns) runs.Clear();
270      Prepare();
271    }
272
273    private bool startPending;
274    public void Start() {
275      Start(CancellationToken.None);
276    }
277    public void Start(CancellationToken cancellationToken) {
278      lock (locker) {
279        if (startPending) return;
280        startPending = true;
281      }
282
283      try {
284        if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
285          throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));
286
287        if (Algorithm == null) return;
288        //create cloned algorithms
289        if (clonedAlgorithms.Count == 0) {
290          int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / Folds.Value;
291
292          for (int i = 0; i < Folds.Value; i++) {
293            IAlgorithm clonedAlgorithm = (IAlgorithm)algorithm.Clone();
294            clonedAlgorithm.Name = algorithm.Name + " Fold " + i;
295            IDataAnalysisProblem problem = clonedAlgorithm.Problem as IDataAnalysisProblem;
296            ISymbolicDataAnalysisProblem symbolicProblem = problem as ISymbolicDataAnalysisProblem;
297
298            int testStart = (i * testSamplesCount) + SamplesStart.Value;
299            int testEnd = (i + 1) == Folds.Value ? SamplesEnd.Value : (i + 1) * testSamplesCount + SamplesStart.Value;
300
301            problem.ProblemData.TrainingPartition.Start = SamplesStart.Value;
302            problem.ProblemData.TrainingPartition.End = SamplesEnd.Value;
303            problem.ProblemData.TestPartition.Start = testStart;
304            problem.ProblemData.TestPartition.End = testEnd;
305            DataAnalysisProblemData problemData = problem.ProblemData as DataAnalysisProblemData;
306            if (problemData != null) {
307              problemData.TrainingPartitionParameter.Hidden = false;
308              problemData.TestPartitionParameter.Hidden = false;
309            }
310
311            if (symbolicProblem != null) {
312              symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
313              symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
314            }
315            clonedAlgorithm.Prepare();
316            clonedAlgorithms.Add(clonedAlgorithm);
317          }
318        }
319
320        OnStarted();
321      } finally {
322        if (startPending) startPending = false;
323      }
324
325      availableWorkers = new SemaphoreSlim(NumberOfWorkers.Value, NumberOfWorkers.Value);
326      allAlgorithmsFinished = new ManualResetEventSlim(false);
327
328      //start prepared or paused cloned algorithms
329      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
330        if (pausePending || stopPending || ExecutionState != ExecutionState.Started) break;
331        if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared ||
332            clonedAlgorithm.ExecutionState == ExecutionState.Paused) {
333          availableWorkers.Wait();
334          lock (locker) {
335            if (pausePending || stopPending || ExecutionState != ExecutionState.Started) break;
336            clonedAlgorithm.StartAsync(cancellationToken);
337          }
338        }
339      }
340
341      allAlgorithmsFinished.Wait();
342    }
343
344    public async Task StartAsync() { await StartAsync(CancellationToken.None); }
345    public async Task StartAsync(CancellationToken cancellationToken) {
346      await Task.Factory.StartNew((ct) => Start((CancellationToken)ct), cancellationToken, cancellationToken);
347    }
348
349    private bool pausePending;
350    public void Pause() {
351      if (startPending) return;
352      if (ExecutionState != ExecutionState.Started)
353        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
354      if (!pausePending) {
355        pausePending = true;
356        lock (locker) {
357          var toPause = clonedAlgorithms.Where(x => x.ExecutionState == ExecutionState.Started);
358          foreach (var optimizer in toPause) {
359            // a race-condition may occur when the optimizer has changed the state by itself in the meantime
360            try { optimizer.Pause(); } catch (InvalidOperationException) { }
361          }
362        }
363      }
364    }
365
366    private bool stopPending;
367    public void Stop() {
368      if (startPending) return;
369      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
370        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".",
371                                                          ExecutionState));
372      if (!stopPending) {
373        stopPending = true;
374        lock (locker) {
375          var toStop = clonedAlgorithms.Where(x => x.ExecutionState == ExecutionState.Started || x.ExecutionState == ExecutionState.Paused);
376          foreach (var optimizer in toStop) {
377            // a race-condition may occur when the optimizer has changed the state by itself in the meantime
378            try { optimizer.Stop(); } catch (InvalidOperationException) { }
379          }
380        }
381      }
382    }
383
384    #region collect parameters and results
385    public override void CollectParameterValues(IDictionary<string, IItem> values) {
386      values.Add("Algorithm Name", new StringValue(Name));
387      values.Add("Algorithm Type", new StringValue(GetType().GetPrettyName()));
388      values.Add("Folds", new IntValue(Folds.Value));
389
390      if (algorithm != null) {
391        values.Add("CrossValidation Algorithm Name", new StringValue(Algorithm.Name));
392        values.Add("CrossValidation Algorithm Type", new StringValue(Algorithm.GetType().GetPrettyName()));
393        base.CollectParameterValues(values);
394      }
395      if (Problem != null) {
396        values.Add("Problem Name", new StringValue(Problem.Name));
397        values.Add("Problem Type", new StringValue(Problem.GetType().GetPrettyName()));
398        Problem.CollectParameterValues(values);
399      }
400    }
401
402    public void CollectResultValues(IDictionary<string, IItem> results) {
403      var clonedResults = (ResultCollection)this.results.Clone();
404      foreach (var result in clonedResults) {
405        results.Add(result.Name, result.Value);
406      }
407    }
408
409    private void AggregateResultValues(IDictionary<string, IItem> results) {
410      IEnumerable<IRun> runs = clonedAlgorithms.Select(alg => alg.Runs.FirstOrDefault()).Where(run => run != null);
411      IEnumerable<KeyValuePair<string, IItem>> resultCollections = runs.Where(x => x != null).SelectMany(x => x.Results).ToList();
412
413      foreach (IResult result in ExtractAndAggregateResults<IntValue>(resultCollections))
414        results.Add(result.Name, result.Value);
415      foreach (IResult result in ExtractAndAggregateResults<DoubleValue>(resultCollections))
416        results.Add(result.Name, result.Value);
417      foreach (IResult result in ExtractAndAggregateResults<PercentValue>(resultCollections))
418        results.Add(result.Name, result.Value);
419      foreach (IResult result in ExtractAndAggregateRegressionSolutions(resultCollections)) {
420        results.Add(result.Name, result.Value);
421      }
422      foreach (IResult result in ExtractAndAggregateClassificationSolutions(resultCollections)) {
423        results.Add(result.Name, result.Value);
424      }
425      results.Add("Execution Time", new TimeSpanValue(this.ExecutionTime));
426      results.Add("CrossValidation Folds", new RunCollection(runs));
427    }
428
429    private IEnumerable<IResult> ExtractAndAggregateRegressionSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
430      Dictionary<string, List<IRegressionSolution>> resultSolutions = new Dictionary<string, List<IRegressionSolution>>();
431      foreach (var result in resultCollections) {
432        var regressionSolution = result.Value as IRegressionSolution;
433        if (regressionSolution != null) {
434          if (resultSolutions.ContainsKey(result.Key)) {
435            resultSolutions[result.Key].Add(regressionSolution);
436          } else {
437            resultSolutions.Add(result.Key, new List<IRegressionSolution>() { regressionSolution });
438          }
439        }
440      }
441      List<IResult> aggregatedResults = new List<IResult>();
442      foreach (KeyValuePair<string, List<IRegressionSolution>> solutions in resultSolutions) {
443        // clone manually to correctly clone references between cloned root objects
444        Cloner cloner = new Cloner();
445        var problemDataClone = (IRegressionProblemData)cloner.Clone(Problem.ProblemData);
446        // set partitions of problem data clone correctly
447        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
448        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
449        // clone models
450        var ensembleSolution = new RegressionEnsembleSolution(problemDataClone);
451        ensembleSolution.AddRegressionSolutions(solutions.Value);
452
453        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
454      }
455      List<IResult> flattenedResults = new List<IResult>();
456      CollectResultsRecursively("", aggregatedResults, flattenedResults);
457      return flattenedResults;
458    }
459
460    private IEnumerable<IResult> ExtractAndAggregateClassificationSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
461      Dictionary<string, List<IClassificationSolution>> resultSolutions = new Dictionary<string, List<IClassificationSolution>>();
462      foreach (var result in resultCollections) {
463        var classificationSolution = result.Value as IClassificationSolution;
464        if (classificationSolution != null) {
465          if (resultSolutions.ContainsKey(result.Key)) {
466            resultSolutions[result.Key].Add(classificationSolution);
467          } else {
468            resultSolutions.Add(result.Key, new List<IClassificationSolution>() { classificationSolution });
469          }
470        }
471      }
472      var aggregatedResults = new List<IResult>();
473      foreach (KeyValuePair<string, List<IClassificationSolution>> solutions in resultSolutions) {
474        // clone manually to correctly clone references between cloned root objects
475        Cloner cloner = new Cloner();
476        var problemDataClone = (IClassificationProblemData)cloner.Clone(Problem.ProblemData);
477        // set partitions of problem data clone correctly
478        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
479        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
480        // clone models
481        var ensembleSolution = new ClassificationEnsembleSolution(problemDataClone);
482        ensembleSolution.AddClassificationSolutions(solutions.Value);
483
484        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
485      }
486      List<IResult> flattenedResults = new List<IResult>();
487      CollectResultsRecursively("", aggregatedResults, flattenedResults);
488      return flattenedResults;
489    }
490
491    private void CollectResultsRecursively(string path, IEnumerable<IResult> results, IList<IResult> flattenedResults) {
492      foreach (IResult result in results) {
493        flattenedResults.Add(new Result(path + result.Name, result.Value));
494        ResultCollection childCollection = result.Value as ResultCollection;
495        if (childCollection != null) {
496          CollectResultsRecursively(path + result.Name + ".", childCollection, flattenedResults);
497        }
498      }
499    }
500
501    private static IEnumerable<IResult> ExtractAndAggregateResults<T>(IEnumerable<KeyValuePair<string, IItem>> results)
502  where T : class, IItem, new() {
503      Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
504      foreach (var resultValue in results.Where(r => r.Value.GetType() == typeof(T))) {
505        if (!resultValues.ContainsKey(resultValue.Key))
506          resultValues[resultValue.Key] = new List<double>();
507        resultValues[resultValue.Key].Add(ConvertToDouble(resultValue.Value));
508      }
509
510      DoubleValue doubleValue;
511      if (typeof(T) == typeof(PercentValue))
512        doubleValue = new PercentValue();
513      else if (typeof(T) == typeof(DoubleValue))
514        doubleValue = new DoubleValue();
515      else if (typeof(T) == typeof(IntValue))
516        doubleValue = new DoubleValue();
517      else
518        throw new NotSupportedException();
519
520      List<IResult> aggregatedResults = new List<IResult>();
521      foreach (KeyValuePair<string, List<double>> resultValue in resultValues) {
522        doubleValue.Value = resultValue.Value.Average();
523        aggregatedResults.Add(new Result(resultValue.Key + " (average)", (IItem)doubleValue.Clone()));
524        doubleValue.Value = resultValue.Value.StandardDeviation();
525        aggregatedResults.Add(new Result(resultValue.Key + " (std.dev.)", (IItem)doubleValue.Clone()));
526      }
527      return aggregatedResults;
528    }
529
530    private static double ConvertToDouble(IItem item) {
531      if (item is DoubleValue) return ((DoubleValue)item).Value;
532      else if (item is IntValue) return ((IntValue)item).Value;
533      else throw new NotSupportedException("Could not convert any item type to double");
534    }
535    #endregion
536
537    #region events
538    private void RegisterEvents() {
539      Folds.ValueChanged += new EventHandler(Folds_ValueChanged);
540      RegisterClonedAlgorithmsEvents();
541    }
542    private void Folds_ValueChanged(object sender, EventArgs e) {
543      if (ExecutionState != ExecutionState.Prepared)
544        throw new InvalidOperationException("Can not change number of folds if the execution state is not prepared.");
545    }
546
547
548    #region template algorithms events
549    public event EventHandler AlgorithmChanged;
550    private void OnAlgorithmChanged() {
551      EventHandler handler = AlgorithmChanged;
552      if (handler != null) handler(this, EventArgs.Empty);
553      OnProblemChanged();
554      if (Problem == null) ExecutionState = ExecutionState.Stopped;
555    }
556    private void RegisterAlgorithmEvents() {
557      algorithm.ProblemChanged += new EventHandler(Algorithm_ProblemChanged);
558      algorithm.ExecutionStateChanged += new EventHandler(Algorithm_ExecutionStateChanged);
559      if (Problem != null) Problem.Reset += new EventHandler(Problem_Reset);
560    }
561    private void DeregisterAlgorithmEvents() {
562      algorithm.ProblemChanged -= new EventHandler(Algorithm_ProblemChanged);
563      algorithm.ExecutionStateChanged -= new EventHandler(Algorithm_ExecutionStateChanged);
564      if (Problem != null) Problem.Reset -= new EventHandler(Problem_Reset);
565    }
566    private void Algorithm_ProblemChanged(object sender, EventArgs e) {
567      if (algorithm.Problem != null && !(algorithm.Problem is IDataAnalysisProblem)) {
568        algorithm.Problem = problem;
569        throw new ArgumentException("A cross validation algorithm can only contain DataAnalysisProblems.");
570      }
571      if (problem != null) problem.Reset -= new EventHandler(Problem_Reset);
572      problem = (IDataAnalysisProblem)algorithm.Problem;
573      if (problem != null) problem.Reset += new EventHandler(Problem_Reset);
574      OnProblemChanged();
575    }
576    public event EventHandler ProblemChanged;
577    private void OnProblemChanged() {
578      EventHandler handler = ProblemChanged;
579      if (handler != null) handler(this, EventArgs.Empty);
580      ConfigureProblem();
581    }
582
583    private void Problem_Reset(object sender, EventArgs e) {
584      ConfigureProblem();
585    }
586
587    private void ConfigureProblem() {
588      SamplesStart.Value = 0;
589      if (Problem != null) {
590        SamplesEnd.Value = Problem.ProblemData.Dataset.Rows;
591
592        DataAnalysisProblemData problemData = Problem.ProblemData as DataAnalysisProblemData;
593        if (problemData != null) {
594          problemData.TrainingPartitionParameter.Hidden = true;
595          problemData.TestPartitionParameter.Hidden = true;
596        }
597        ISymbolicDataAnalysisProblem symbolicProblem = Problem as ISymbolicDataAnalysisProblem;
598        if (symbolicProblem != null) {
599          symbolicProblem.FitnessCalculationPartitionParameter.Hidden = true;
600          symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
601          symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
602          symbolicProblem.ValidationPartitionParameter.Hidden = true;
603          symbolicProblem.ValidationPartition.Start = 0;
604          symbolicProblem.ValidationPartition.End = 0;
605        }
606      } else
607        SamplesEnd.Value = 0;
608    }
609
610    private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
611      switch (Algorithm.ExecutionState) {
612        case ExecutionState.Prepared:
613          OnPrepared();
614          break;
615        case ExecutionState.Started: throw new InvalidOperationException("Algorithm template can not be started.");
616        case ExecutionState.Paused: throw new InvalidOperationException("Algorithm template can not be paused.");
617        case ExecutionState.Stopped:
618          OnStopped();
619          break;
620      }
621    }
622    #endregion
623
624    #region clonedAlgorithms events
625    private void RegisterClonedAlgorithmsEvents() {
626      clonedAlgorithms.ItemsAdded += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
627      clonedAlgorithms.ItemsRemoved += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
628      clonedAlgorithms.CollectionReset += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
629      foreach (IAlgorithm algorithm in clonedAlgorithms)
630        RegisterClonedAlgorithmEvents(algorithm);
631    }
632    private void DeregisterClonedAlgorithmsEvents() {
633      clonedAlgorithms.ItemsAdded -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
634      clonedAlgorithms.ItemsRemoved -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
635      clonedAlgorithms.CollectionReset -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
636      foreach (IAlgorithm algorithm in clonedAlgorithms)
637        DeregisterClonedAlgorithmEvents(algorithm);
638    }
639    private void ClonedAlgorithms_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
640      foreach (IAlgorithm algorithm in e.Items)
641        RegisterClonedAlgorithmEvents(algorithm);
642    }
643    private void ClonedAlgorithms_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
644      foreach (IAlgorithm algorithm in e.Items)
645        DeregisterClonedAlgorithmEvents(algorithm);
646    }
647    private void ClonedAlgorithms_CollectionReset(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
648      foreach (IAlgorithm algorithm in e.OldItems)
649        DeregisterClonedAlgorithmEvents(algorithm);
650      foreach (IAlgorithm algorithm in e.Items)
651        RegisterClonedAlgorithmEvents(algorithm);
652    }
653    private void RegisterClonedAlgorithmEvents(IAlgorithm algorithm) {
654      algorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
655      algorithm.ExecutionTimeChanged += new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
656      algorithm.Started += new EventHandler(ClonedAlgorithm_Started);
657      algorithm.Paused += new EventHandler(ClonedAlgorithm_Paused);
658      algorithm.Stopped += new EventHandler(ClonedAlgorithm_Stopped);
659    }
660    private void DeregisterClonedAlgorithmEvents(IAlgorithm algorithm) {
661      algorithm.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
662      algorithm.ExecutionTimeChanged -= new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
663      algorithm.Started -= new EventHandler(ClonedAlgorithm_Started);
664      algorithm.Paused -= new EventHandler(ClonedAlgorithm_Paused);
665      algorithm.Stopped -= new EventHandler(ClonedAlgorithm_Stopped);
666    }
667    private void ClonedAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
668      Pause();
669      OnExceptionOccurred(e.Value);
670    }
671    private void ClonedAlgorithm_ExecutionTimeChanged(object sender, EventArgs e) {
672      OnExecutionTimeChanged();
673    }
674
675    private readonly object locker = new object();
676    private readonly object resultLocker = new object();
677    private void ClonedAlgorithm_Started(object sender, EventArgs e) {
678      IAlgorithm algorithm = sender as IAlgorithm;
679      lock (resultLocker) {
680        if (algorithm != null && !results.ContainsKey(algorithm.Name))
681          results.Add(new Result(algorithm.Name, "Contains results for the specific fold.", algorithm.Results));
682      }
683    }
684
685    private void ClonedAlgorithm_Paused(object sender, EventArgs e) {
686      lock (locker) {
687        availableWorkers.Release();
688        if (clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started)) {
689          OnPaused();
690          allAlgorithmsFinished.Set();
691        }
692      }
693    }
694
695    private void ClonedAlgorithm_Stopped(object sender, EventArgs e) {
696      lock (locker) {
697        // if the algorithm was in paused state, its worker has already been released
698        if (availableWorkers.CurrentCount < NumberOfWorkers.Value)
699          availableWorkers.Release();
700        if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped)) {
701          OnStopped();
702          allAlgorithmsFinished.Set();
703        } else if (stopPending && clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped)) {
704          OnStopped();
705          allAlgorithmsFinished.Set();
706        }
707      }
708    }
709    #endregion
710    #endregion
711
712    #region event firing
713    public event EventHandler ExecutionStateChanged;
714    private void OnExecutionStateChanged() {
715      EventHandler handler = ExecutionStateChanged;
716      if (handler != null) handler(this, EventArgs.Empty);
717    }
718    public event EventHandler ExecutionTimeChanged;
719    private void OnExecutionTimeChanged() {
720      EventHandler handler = ExecutionTimeChanged;
721      if (handler != null) handler(this, EventArgs.Empty);
722    }
723    public event EventHandler Prepared;
724    private void OnPrepared() {
725      ExecutionState = ExecutionState.Prepared;
726      EventHandler handler = Prepared;
727      if (handler != null) handler(this, EventArgs.Empty);
728      OnExecutionTimeChanged();
729    }
730    public event EventHandler Started;
731    private void OnStarted() {
732      startPending = false;
733      ExecutionState = ExecutionState.Started;
734      EventHandler handler = Started;
735      if (handler != null) handler(this, EventArgs.Empty);
736    }
737    public event EventHandler Paused;
738    private void OnPaused() {
739      pausePending = false;
740      ExecutionState = ExecutionState.Paused;
741      EventHandler handler = Paused;
742      if (handler != null) handler(this, EventArgs.Empty);
743    }
744    public event EventHandler Stopped;
745    private void OnStopped() {
746      stopPending = false;
747      Dictionary<string, IItem> collectedResults = new Dictionary<string, IItem>();
748      AggregateResultValues(collectedResults);
749      results.AddRange(collectedResults.Select(x => new Result(x.Key, x.Value)).Cast<IResult>().ToArray());
750      runsCounter++;
751      runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
752      ExecutionState = ExecutionState.Stopped;
753      EventHandler handler = Stopped;
754      if (handler != null) handler(this, EventArgs.Empty);
755    }
756    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
757    private void OnExceptionOccurred(Exception exception) {
758      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
759      if (handler != null) handler(this, new EventArgs<Exception>(exception));
760    }
761    public event EventHandler StoreAlgorithmInEachRunChanged;
762    private void OnStoreAlgorithmInEachRunChanged() {
763      EventHandler handler = StoreAlgorithmInEachRunChanged;
764      if (handler != null) handler(this, EventArgs.Empty);
765    }
766    #endregion
767  }
768}
Note: See TracBrowser for help on using the repository browser.