source: branches/Async/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs @ 13349

Last change on this file since 13349 was 13349, checked in by jkarder, 4 years ago

#2258: added StartAsync to IExecutable

File size: 32.5 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Drawing;
25using System.Linq;
26using System.Threading;
27using System.Threading.Tasks;
28using HeuristicLab.Collections;
29using HeuristicLab.Common;
30using HeuristicLab.Core;
31using HeuristicLab.Data;
32using HeuristicLab.Optimization;
33using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
34using HeuristicLab.Problems.DataAnalysis;
35using HeuristicLab.Problems.DataAnalysis.Symbolic;
36
37namespace HeuristicLab.Algorithms.DataAnalysis {
38  [Item("Cross Validation (CV)", "Cross-validation wrapper for data analysis algorithms.")]
39  [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 100)]
40  [StorableClass]
41  public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent {
42    private readonly ManualResetEvent signaler = new ManualResetEvent(true);
43
44    public CrossValidation()
45      : base() {
46      name = ItemName;
47      description = ItemDescription;
48
49      executionState = ExecutionState.Stopped;
50      runs = new RunCollection { OptimizerName = name };
51      runsCounter = 0;
52
53      algorithm = null;
54      clonedAlgorithms = new ItemCollection<IAlgorithm>();
55      results = new ResultCollection();
56
57      folds = new IntValue(2);
58      numberOfWorkers = new IntValue(1);
59      samplesStart = new IntValue(0);
60      samplesEnd = new IntValue(0);
61      storeAlgorithmInEachRun = false;
62
63      RegisterEvents();
64      if (Algorithm != null) RegisterAlgorithmEvents();
65    }
66
67    public string Filename { get; set; }
68
69    #region persistence and cloning
70    [StorableConstructor]
71    private CrossValidation(bool deserializing)
72      : base(deserializing) {
73    }
74    [StorableHook(HookType.AfterDeserialization)]
75    private void AfterDeserialization() {
76      RegisterEvents();
77      if (Algorithm != null) RegisterAlgorithmEvents();
78    }
79
80    private CrossValidation(CrossValidation original, Cloner cloner)
81      : base(original, cloner) {
82      executionState = original.executionState;
83      storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
84      runs = cloner.Clone(original.runs);
85      runsCounter = original.runsCounter;
86      algorithm = cloner.Clone(original.algorithm);
87      clonedAlgorithms = cloner.Clone(original.clonedAlgorithms);
88      results = cloner.Clone(original.results);
89
90      folds = cloner.Clone(original.folds);
91      numberOfWorkers = cloner.Clone(original.numberOfWorkers);
92      samplesStart = cloner.Clone(original.samplesStart);
93      samplesEnd = cloner.Clone(original.samplesEnd);
94      RegisterEvents();
95      if (Algorithm != null) RegisterAlgorithmEvents();
96    }
97    public override IDeepCloneable Clone(Cloner cloner) {
98      return new CrossValidation(this, cloner);
99    }
100
101    #endregion
102
103    #region properties
104    [Storable]
105    private IAlgorithm algorithm;
106    public IAlgorithm Algorithm {
107      get { return algorithm; }
108      set {
109        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
110          throw new InvalidOperationException("Changing the algorithm is only allowed if the CrossValidation is stopped or prepared.");
111        if (algorithm != value) {
112          if (value != null && value.Problem != null && !(value.Problem is IDataAnalysisProblem))
113            throw new ArgumentException("Only algorithms with a DataAnalysisProblem could be used for the cross validation.");
114          if (algorithm != null) DeregisterAlgorithmEvents();
115          algorithm = value;
116          Parameters.Clear();
117
118          if (algorithm != null) {
119            algorithm.StoreAlgorithmInEachRun = false;
120            RegisterAlgorithmEvents();
121            algorithm.Prepare(true);
122            Parameters.AddRange(algorithm.Parameters);
123          }
124          OnAlgorithmChanged();
125          Prepare();
126        }
127      }
128    }
129
130
131    [Storable]
132    private IDataAnalysisProblem problem;
133    public IDataAnalysisProblem Problem {
134      get {
135        if (algorithm == null)
136          return null;
137        return (IDataAnalysisProblem)algorithm.Problem;
138      }
139      set {
140        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
141          throw new InvalidOperationException("Changing the problem is only allowed if the CrossValidation is stopped or prepared.");
142        if (algorithm == null) throw new ArgumentNullException("Could not set a problem before an algorithm was set.");
143        algorithm.Problem = value;
144        problem = value;
145      }
146    }
147
148    IProblem IAlgorithm.Problem {
149      get { return Problem; }
150      set {
151        if (value != null && !ProblemType.IsInstanceOfType(value))
152          throw new ArgumentException("Only DataAnalysisProblems could be used for the cross validation.");
153        Problem = (IDataAnalysisProblem)value;
154      }
155    }
156    public Type ProblemType {
157      get { return typeof(IDataAnalysisProblem); }
158    }
159
160    [Storable]
161    private ItemCollection<IAlgorithm> clonedAlgorithms;
162
163    public IEnumerable<IOptimizer> NestedOptimizers {
164      get {
165        if (Algorithm == null) yield break;
166        yield return Algorithm;
167      }
168    }
169
170    [Storable]
171    private ResultCollection results;
172    public ResultCollection Results {
173      get { return results; }
174    }
175
176    [Storable]
177    private IntValue folds;
178    public IntValue Folds {
179      get { return folds; }
180    }
181    [Storable]
182    private IntValue samplesStart;
183    public IntValue SamplesStart {
184      get { return samplesStart; }
185    }
186    [Storable]
187    private IntValue samplesEnd;
188    public IntValue SamplesEnd {
189      get { return samplesEnd; }
190    }
191    [Storable]
192    private IntValue numberOfWorkers;
193    public IntValue NumberOfWorkers {
194      get { return numberOfWorkers; }
195    }
196
197    [Storable]
198    private bool storeAlgorithmInEachRun;
199    public bool StoreAlgorithmInEachRun {
200      get { return storeAlgorithmInEachRun; }
201      set {
202        if (storeAlgorithmInEachRun != value) {
203          storeAlgorithmInEachRun = value;
204          OnStoreAlgorithmInEachRunChanged();
205        }
206      }
207    }
208
209    [Storable]
210    private int runsCounter;
211    [Storable]
212    private RunCollection runs;
213    public RunCollection Runs {
214      get { return runs; }
215    }
216
217    [Storable]
218    private ExecutionState executionState;
219    public ExecutionState ExecutionState {
220      get { return executionState; }
221      private set {
222        if (executionState != value) {
223          executionState = value;
224          OnExecutionStateChanged();
225          OnItemImageChanged();
226        }
227      }
228    }
229    public static new Image StaticItemImage {
230      get { return HeuristicLab.Common.Resources.VSImageLibrary.Event; }
231    }
232    public override Image ItemImage {
233      get {
234        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
235        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
236        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
237        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
238        else return base.ItemImage;
239      }
240    }
241
242    public TimeSpan ExecutionTime {
243      get {
244        if (ExecutionState != ExecutionState.Prepared)
245          return TimeSpan.FromMilliseconds(clonedAlgorithms.Select(x => x.ExecutionTime.TotalMilliseconds).Sum());
246        return TimeSpan.Zero;
247      }
248    }
249    #endregion
250
251    protected override void OnNameChanged() {
252      base.OnNameChanged();
253      Runs.OptimizerName = Name;
254    }
255
256    public void Prepare() {
257      if (ExecutionState == ExecutionState.Started)
258        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
259      results.Clear();
260      clonedAlgorithms.Clear();
261      if (Algorithm != null) {
262        Algorithm.Prepare();
263        if (Algorithm.ExecutionState == ExecutionState.Prepared) OnPrepared();
264      }
265    }
266    public void Prepare(bool clearRuns) {
267      if (clearRuns) runs.Clear();
268      Prepare();
269    }
270    public void Start() {
271      StartAsync().Wait();
272    }
273    public async Task StartAsync() {
274      await StartAsync(new CancellationToken());
275    }
276    public async Task StartAsync(CancellationToken cancellationToken) {
277      signaler.Reset();
278      await Task.Run(() => {
279        if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
280          throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));
281
282        if (Algorithm != null) {
283          //create cloned algorithms
284          if (clonedAlgorithms.Count == 0) {
285            int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / Folds.Value;
286
287            for (int i = 0; i < Folds.Value; i++) {
288              IAlgorithm clonedAlgorithm = (IAlgorithm)algorithm.Clone();
289              clonedAlgorithm.Name = algorithm.Name + " Fold " + i;
290              IDataAnalysisProblem problem = clonedAlgorithm.Problem as IDataAnalysisProblem;
291              ISymbolicDataAnalysisProblem symbolicProblem = problem as ISymbolicDataAnalysisProblem;
292
293              int testStart = (i * testSamplesCount) + SamplesStart.Value;
294              int testEnd = (i + 1) == Folds.Value ? SamplesEnd.Value : (i + 1) * testSamplesCount + SamplesStart.Value;
295
296              problem.ProblemData.TrainingPartition.Start = SamplesStart.Value;
297              problem.ProblemData.TrainingPartition.End = SamplesEnd.Value;
298              problem.ProblemData.TestPartition.Start = testStart;
299              problem.ProblemData.TestPartition.End = testEnd;
300              DataAnalysisProblemData problemData = problem.ProblemData as DataAnalysisProblemData;
301              if (problemData != null) {
302                problemData.TrainingPartitionParameter.Hidden = false;
303                problemData.TestPartitionParameter.Hidden = false;
304              }
305
306              if (symbolicProblem != null) {
307                symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
308                symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
309              }
310              clonedAlgorithm.Prepare();
311              clonedAlgorithms.Add(clonedAlgorithm);
312            }
313          }
314
315          //start prepared or paused cloned algorithms
316          int startedAlgorithms = 0;
317          foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
318            if (startedAlgorithms < NumberOfWorkers.Value) {
319              if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared ||
320                  clonedAlgorithm.ExecutionState == ExecutionState.Paused) {
321
322                // start and wait until the alg is started
323                using (var signal = new ManualResetEvent(false)) {
324                  EventHandler signalSetter = (sender, args) => { signal.Set(); };
325                  clonedAlgorithm.Started += signalSetter;
326                  clonedAlgorithm.StartAsync(cancellationToken);
327                  signal.WaitOne();
328                  clonedAlgorithm.Started -= signalSetter;
329
330                  startedAlgorithms++;
331                }
332              }
333            }
334          }
335          OnStarted();
336        }
337      }, cancellationToken);
338    }
339
340    private bool pausePending;
341    public void Pause() {
342      if (ExecutionState != ExecutionState.Started)
343        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
344      if (!pausePending) {
345        pausePending = true;
346        PauseAllClonedAlgorithms();
347      }
348    }
349    private void PauseAllClonedAlgorithms() {
350      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
351        if (clonedAlgorithm.ExecutionState == ExecutionState.Started)
352          clonedAlgorithm.Pause();
353      }
354    }
355
356    private bool stopPending;
357    public void Stop() {
358      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
359        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".",
360                                                          ExecutionState));
361      if (!stopPending) {
362        stopPending = true;
363        StopAllClonedAlgorithms();
364      }
365    }
366    private void StopAllClonedAlgorithms() {
367      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
368        if (clonedAlgorithm.ExecutionState == ExecutionState.Started ||
369            clonedAlgorithm.ExecutionState == ExecutionState.Paused)
370          clonedAlgorithm.Stop();
371      }
372    }
373
374    #region collect parameters and results
375    public override void CollectParameterValues(IDictionary<string, IItem> values) {
376      values.Add("Algorithm Name", new StringValue(Name));
377      values.Add("Algorithm Type", new StringValue(GetType().GetPrettyName()));
378      values.Add("Folds", new IntValue(Folds.Value));
379
380      if (algorithm != null) {
381        values.Add("CrossValidation Algorithm Name", new StringValue(Algorithm.Name));
382        values.Add("CrossValidation Algorithm Type", new StringValue(Algorithm.GetType().GetPrettyName()));
383        base.CollectParameterValues(values);
384      }
385      if (Problem != null) {
386        values.Add("Problem Name", new StringValue(Problem.Name));
387        values.Add("Problem Type", new StringValue(Problem.GetType().GetPrettyName()));
388        Problem.CollectParameterValues(values);
389      }
390    }
391
392    public void CollectResultValues(IDictionary<string, IItem> results) {
393      var clonedResults = (ResultCollection)this.results.Clone();
394      foreach (var result in clonedResults) {
395        results.Add(result.Name, result.Value);
396      }
397    }
398
399    private void AggregateResultValues(IDictionary<string, IItem> results) {
400      IEnumerable<IRun> runs = clonedAlgorithms.Select(alg => alg.Runs.FirstOrDefault()).Where(run => run != null);
401      IEnumerable<KeyValuePair<string, IItem>> resultCollections = runs.Where(x => x != null).SelectMany(x => x.Results).ToList();
402
403      foreach (IResult result in ExtractAndAggregateResults<IntValue>(resultCollections))
404        results.Add(result.Name, result.Value);
405      foreach (IResult result in ExtractAndAggregateResults<DoubleValue>(resultCollections))
406        results.Add(result.Name, result.Value);
407      foreach (IResult result in ExtractAndAggregateResults<PercentValue>(resultCollections))
408        results.Add(result.Name, result.Value);
409      foreach (IResult result in ExtractAndAggregateRegressionSolutions(resultCollections)) {
410        results.Add(result.Name, result.Value);
411      }
412      foreach (IResult result in ExtractAndAggregateClassificationSolutions(resultCollections)) {
413        results.Add(result.Name, result.Value);
414      }
415      results.Add("Execution Time", new TimeSpanValue(this.ExecutionTime));
416      results.Add("CrossValidation Folds", new RunCollection(runs));
417    }
418
419    private IEnumerable<IResult> ExtractAndAggregateRegressionSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
420      Dictionary<string, List<IRegressionSolution>> resultSolutions = new Dictionary<string, List<IRegressionSolution>>();
421      foreach (var result in resultCollections) {
422        var regressionSolution = result.Value as IRegressionSolution;
423        if (regressionSolution != null) {
424          if (resultSolutions.ContainsKey(result.Key)) {
425            resultSolutions[result.Key].Add(regressionSolution);
426          } else {
427            resultSolutions.Add(result.Key, new List<IRegressionSolution>() { regressionSolution });
428          }
429        }
430      }
431      List<IResult> aggregatedResults = new List<IResult>();
432      foreach (KeyValuePair<string, List<IRegressionSolution>> solutions in resultSolutions) {
433        // clone manually to correctly clone references between cloned root objects
434        Cloner cloner = new Cloner();
435        var problemDataClone = (IRegressionProblemData)cloner.Clone(Problem.ProblemData);
436        // set partitions of problem data clone correctly
437        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
438        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
439        // clone models
440        var ensembleSolution = new RegressionEnsembleSolution(problemDataClone);
441        ensembleSolution.AddRegressionSolutions(solutions.Value);
442
443        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
444      }
445      List<IResult> flattenedResults = new List<IResult>();
446      CollectResultsRecursively("", aggregatedResults, flattenedResults);
447      return flattenedResults;
448    }
449
450    private IEnumerable<IResult> ExtractAndAggregateClassificationSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
451      Dictionary<string, List<IClassificationSolution>> resultSolutions = new Dictionary<string, List<IClassificationSolution>>();
452      foreach (var result in resultCollections) {
453        var classificationSolution = result.Value as IClassificationSolution;
454        if (classificationSolution != null) {
455          if (resultSolutions.ContainsKey(result.Key)) {
456            resultSolutions[result.Key].Add(classificationSolution);
457          } else {
458            resultSolutions.Add(result.Key, new List<IClassificationSolution>() { classificationSolution });
459          }
460        }
461      }
462      var aggregatedResults = new List<IResult>();
463      foreach (KeyValuePair<string, List<IClassificationSolution>> solutions in resultSolutions) {
464        // clone manually to correctly clone references between cloned root objects
465        Cloner cloner = new Cloner();
466        var problemDataClone = (IClassificationProblemData)cloner.Clone(Problem.ProblemData);
467        // set partitions of problem data clone correctly
468        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
469        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
470        // clone models
471        var ensembleSolution = new ClassificationEnsembleSolution(problemDataClone);
472        ensembleSolution.AddClassificationSolutions(solutions.Value);
473
474        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
475      }
476      List<IResult> flattenedResults = new List<IResult>();
477      CollectResultsRecursively("", aggregatedResults, flattenedResults);
478      return flattenedResults;
479    }
480
481    private void CollectResultsRecursively(string path, IEnumerable<IResult> results, IList<IResult> flattenedResults) {
482      foreach (IResult result in results) {
483        flattenedResults.Add(new Result(path + result.Name, result.Value));
484        ResultCollection childCollection = result.Value as ResultCollection;
485        if (childCollection != null) {
486          CollectResultsRecursively(path + result.Name + ".", childCollection, flattenedResults);
487        }
488      }
489    }
490
491    private static IEnumerable<IResult> ExtractAndAggregateResults<T>(IEnumerable<KeyValuePair<string, IItem>> results)
492  where T : class, IItem, new() {
493      Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
494      foreach (var resultValue in results.Where(r => r.Value.GetType() == typeof(T))) {
495        if (!resultValues.ContainsKey(resultValue.Key))
496          resultValues[resultValue.Key] = new List<double>();
497        resultValues[resultValue.Key].Add(ConvertToDouble(resultValue.Value));
498      }
499
500      DoubleValue doubleValue;
501      if (typeof(T) == typeof(PercentValue))
502        doubleValue = new PercentValue();
503      else if (typeof(T) == typeof(DoubleValue))
504        doubleValue = new DoubleValue();
505      else if (typeof(T) == typeof(IntValue))
506        doubleValue = new DoubleValue();
507      else
508        throw new NotSupportedException();
509
510      List<IResult> aggregatedResults = new List<IResult>();
511      foreach (KeyValuePair<string, List<double>> resultValue in resultValues) {
512        doubleValue.Value = resultValue.Value.Average();
513        aggregatedResults.Add(new Result(resultValue.Key + " (average)", (IItem)doubleValue.Clone()));
514        doubleValue.Value = resultValue.Value.StandardDeviation();
515        aggregatedResults.Add(new Result(resultValue.Key + " (std.dev.)", (IItem)doubleValue.Clone()));
516      }
517      return aggregatedResults;
518    }
519
520    private static double ConvertToDouble(IItem item) {
521      if (item is DoubleValue) return ((DoubleValue)item).Value;
522      else if (item is IntValue) return ((IntValue)item).Value;
523      else throw new NotSupportedException("Could not convert any item type to double");
524    }
525    #endregion
526
527    #region events
528    private void RegisterEvents() {
529      Folds.ValueChanged += new EventHandler(Folds_ValueChanged);
530      RegisterClonedAlgorithmsEvents();
531    }
532    private void Folds_ValueChanged(object sender, EventArgs e) {
533      if (ExecutionState != ExecutionState.Prepared)
534        throw new InvalidOperationException("Can not change number of folds if the execution state is not prepared.");
535    }
536
537
538    #region template algorithms events
539    public event EventHandler AlgorithmChanged;
540    private void OnAlgorithmChanged() {
541      EventHandler handler = AlgorithmChanged;
542      if (handler != null) handler(this, EventArgs.Empty);
543      OnProblemChanged();
544      if (Problem == null) ExecutionState = ExecutionState.Stopped;
545    }
546    private void RegisterAlgorithmEvents() {
547      algorithm.ProblemChanged += new EventHandler(Algorithm_ProblemChanged);
548      algorithm.ExecutionStateChanged += new EventHandler(Algorithm_ExecutionStateChanged);
549      if (Problem != null) Problem.Reset += new EventHandler(Problem_Reset);
550    }
551    private void DeregisterAlgorithmEvents() {
552      algorithm.ProblemChanged -= new EventHandler(Algorithm_ProblemChanged);
553      algorithm.ExecutionStateChanged -= new EventHandler(Algorithm_ExecutionStateChanged);
554      if (Problem != null) Problem.Reset -= new EventHandler(Problem_Reset);
555    }
556    private void Algorithm_ProblemChanged(object sender, EventArgs e) {
557      if (algorithm.Problem != null && !(algorithm.Problem is IDataAnalysisProblem)) {
558        algorithm.Problem = problem;
559        throw new ArgumentException("A cross validation algorithm can only contain DataAnalysisProblems.");
560      }
561      if (problem != null) problem.Reset -= new EventHandler(Problem_Reset);
562      problem = (IDataAnalysisProblem)algorithm.Problem;
563      if (problem != null) problem.Reset += new EventHandler(Problem_Reset);
564      OnProblemChanged();
565    }
566    public event EventHandler ProblemChanged;
567    private void OnProblemChanged() {
568      EventHandler handler = ProblemChanged;
569      if (handler != null) handler(this, EventArgs.Empty);
570      ConfigureProblem();
571    }
572
573    private void Problem_Reset(object sender, EventArgs e) {
574      ConfigureProblem();
575    }
576
577    private void ConfigureProblem() {
578      SamplesStart.Value = 0;
579      if (Problem != null) {
580        SamplesEnd.Value = Problem.ProblemData.Dataset.Rows;
581
582        DataAnalysisProblemData problemData = Problem.ProblemData as DataAnalysisProblemData;
583        if (problemData != null) {
584          problemData.TrainingPartitionParameter.Hidden = true;
585          problemData.TestPartitionParameter.Hidden = true;
586        }
587        ISymbolicDataAnalysisProblem symbolicProblem = Problem as ISymbolicDataAnalysisProblem;
588        if (symbolicProblem != null) {
589          symbolicProblem.FitnessCalculationPartitionParameter.Hidden = true;
590          symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
591          symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
592          symbolicProblem.ValidationPartitionParameter.Hidden = true;
593          symbolicProblem.ValidationPartition.Start = 0;
594          symbolicProblem.ValidationPartition.End = 0;
595        }
596      } else
597        SamplesEnd.Value = 0;
598    }
599
600    private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
601      switch (Algorithm.ExecutionState) {
602        case ExecutionState.Prepared: OnPrepared();
603          break;
604        case ExecutionState.Started: throw new InvalidOperationException("Algorithm template can not be started.");
605        case ExecutionState.Paused: throw new InvalidOperationException("Algorithm template can not be paused.");
606        case ExecutionState.Stopped: OnStopped();
607          break;
608      }
609    }
610    #endregion
611
612    #region clonedAlgorithms events
613    private void RegisterClonedAlgorithmsEvents() {
614      clonedAlgorithms.ItemsAdded += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
615      clonedAlgorithms.ItemsRemoved += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
616      clonedAlgorithms.CollectionReset += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
617      foreach (IAlgorithm algorithm in clonedAlgorithms)
618        RegisterClonedAlgorithmEvents(algorithm);
619    }
620    private void DeregisterClonedAlgorithmsEvents() {
621      clonedAlgorithms.ItemsAdded -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
622      clonedAlgorithms.ItemsRemoved -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
623      clonedAlgorithms.CollectionReset -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
624      foreach (IAlgorithm algorithm in clonedAlgorithms)
625        DeregisterClonedAlgorithmEvents(algorithm);
626    }
627    private void ClonedAlgorithms_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
628      foreach (IAlgorithm algorithm in e.Items)
629        RegisterClonedAlgorithmEvents(algorithm);
630    }
631    private void ClonedAlgorithms_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
632      foreach (IAlgorithm algorithm in e.Items)
633        DeregisterClonedAlgorithmEvents(algorithm);
634    }
635    private void ClonedAlgorithms_CollectionReset(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
636      foreach (IAlgorithm algorithm in e.OldItems)
637        DeregisterClonedAlgorithmEvents(algorithm);
638      foreach (IAlgorithm algorithm in e.Items)
639        RegisterClonedAlgorithmEvents(algorithm);
640    }
641    private void RegisterClonedAlgorithmEvents(IAlgorithm algorithm) {
642      algorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
643      algorithm.ExecutionTimeChanged += new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
644      algorithm.Started += new EventHandler(ClonedAlgorithm_Started);
645      algorithm.Paused += new EventHandler(ClonedAlgorithm_Paused);
646      algorithm.Stopped += new EventHandler(ClonedAlgorithm_Stopped);
647    }
648    private void DeregisterClonedAlgorithmEvents(IAlgorithm algorithm) {
649      algorithm.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
650      algorithm.ExecutionTimeChanged -= new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
651      algorithm.Started -= new EventHandler(ClonedAlgorithm_Started);
652      algorithm.Paused -= new EventHandler(ClonedAlgorithm_Paused);
653      algorithm.Stopped -= new EventHandler(ClonedAlgorithm_Stopped);
654    }
655    private void ClonedAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
656      OnExceptionOccurred(e.Value);
657    }
658    private void ClonedAlgorithm_ExecutionTimeChanged(object sender, EventArgs e) {
659      OnExecutionTimeChanged();
660    }
661
662    private readonly object locker = new object();
663    private readonly object resultLocker = new object();
664    private void ClonedAlgorithm_Started(object sender, EventArgs e) {
665      IAlgorithm algorithm = sender as IAlgorithm;
666      lock (resultLocker) {
667        if (algorithm != null && !results.ContainsKey(algorithm.Name))
668          results.Add(new Result(algorithm.Name, "Contains results for the specific fold.", algorithm.Results));
669      }
670    }
671
672    private void ClonedAlgorithm_Paused(object sender, EventArgs e) {
673      lock (locker) {
674        if (pausePending && clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started))
675          OnPaused();
676      }
677    }
678
679    private void ClonedAlgorithm_Stopped(object sender, EventArgs e) {
680      lock (locker) {
681        if (!stopPending && ExecutionState == ExecutionState.Started) {
682          IAlgorithm preparedAlgorithm = clonedAlgorithms.FirstOrDefault(alg => alg.ExecutionState == ExecutionState.Prepared ||
683                                                                                alg.ExecutionState == ExecutionState.Paused);
684          if (preparedAlgorithm != null) preparedAlgorithm.StartAsync();
685        }
686        if (ExecutionState != ExecutionState.Stopped) {
687          if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped))
688            OnStopped();
689          else if (stopPending &&
690                   clonedAlgorithms.All(
691                     alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped))
692            OnStopped();
693        }
694      }
695    }
696    #endregion
697    #endregion
698
699    #region event firing
700    public event EventHandler ExecutionStateChanged;
701    private void OnExecutionStateChanged() {
702      EventHandler handler = ExecutionStateChanged;
703      if (handler != null) handler(this, EventArgs.Empty);
704    }
705    public event EventHandler ExecutionTimeChanged;
706    private void OnExecutionTimeChanged() {
707      EventHandler handler = ExecutionTimeChanged;
708      if (handler != null) handler(this, EventArgs.Empty);
709    }
710    public event EventHandler Prepared;
711    private void OnPrepared() {
712      ExecutionState = ExecutionState.Prepared;
713      EventHandler handler = Prepared;
714      if (handler != null) handler(this, EventArgs.Empty);
715      OnExecutionTimeChanged();
716    }
717    public event EventHandler Started;
718    private void OnStarted() {
719      ExecutionState = ExecutionState.Started;
720      EventHandler handler = Started;
721      if (handler != null) handler(this, EventArgs.Empty);
722    }
723    public event EventHandler Paused;
724    private void OnPaused() {
725      pausePending = false;
726      ExecutionState = ExecutionState.Paused;
727      signaler.Set();
728      EventHandler handler = Paused;
729      if (handler != null) handler(this, EventArgs.Empty);
730    }
731    public event EventHandler Stopped;
732    private void OnStopped() {
733      stopPending = false;
734      Dictionary<string, IItem> collectedResults = new Dictionary<string, IItem>();
735      AggregateResultValues(collectedResults);
736      results.AddRange(collectedResults.Select(x => new Result(x.Key, x.Value)).Cast<IResult>().ToArray());
737      runsCounter++;
738      runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
739      ExecutionState = ExecutionState.Stopped;
740      signaler.Set();
741      EventHandler handler = Stopped;
742      if (handler != null) handler(this, EventArgs.Empty);
743    }
744    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
745    private void OnExceptionOccurred(Exception exception) {
746      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
747      if (handler != null) handler(this, new EventArgs<Exception>(exception));
748    }
749    public event EventHandler StoreAlgorithmInEachRunChanged;
750    private void OnStoreAlgorithmInEachRunChanged() {
751      EventHandler handler = StoreAlgorithmInEachRunChanged;
752      if (handler != null) handler(this, EventArgs.Empty);
753    }
754    #endregion
755  }
756}
Note: See TracBrowser for help on using the repository browser.