source: branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs @ 14869

Last change on this file since 14869 was 14869, checked in by gkronber, 2 years ago

#2699: merged changesets from trunk to branch

File size: 32.2 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Drawing;
25using System.Linq;
26using System.Threading;
27using HeuristicLab.Collections;
28using HeuristicLab.Common;
29using HeuristicLab.Core;
30using HeuristicLab.Data;
31using HeuristicLab.Optimization;
32using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
33using HeuristicLab.Problems.DataAnalysis;
34using HeuristicLab.Problems.DataAnalysis.Symbolic;
35
36namespace HeuristicLab.Algorithms.DataAnalysis {
37  [Item("Cross Validation (CV)", "Cross-validation wrapper for data analysis algorithms.")]
38  [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 100)]
39  [StorableClass]
40  public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent {
41    public CrossValidation()
42      : base() {
43      name = ItemName;
44      description = ItemDescription;
45
46      executionState = ExecutionState.Stopped;
47      runs = new RunCollection { OptimizerName = name };
48      runsCounter = 0;
49
50      algorithm = null;
51      clonedAlgorithms = new ItemCollection<IAlgorithm>();
52      results = new ResultCollection();
53
54      folds = new IntValue(2);
55      numberOfWorkers = new IntValue(1);
56      samplesStart = new IntValue(0);
57      samplesEnd = new IntValue(0);
58      storeAlgorithmInEachRun = false;
59
60      RegisterEvents();
61      if (Algorithm != null) RegisterAlgorithmEvents();
62    }
63
64    public string Filename { get; set; }
65
66    #region persistence and cloning
67    [StorableConstructor]
68    private CrossValidation(bool deserializing)
69      : base(deserializing) {
70    }
71    [StorableHook(HookType.AfterDeserialization)]
72    private void AfterDeserialization() {
73      RegisterEvents();
74      if (Algorithm != null) RegisterAlgorithmEvents();
75    }
76
77    private CrossValidation(CrossValidation original, Cloner cloner)
78      : base(original, cloner) {
79      executionState = original.executionState;
80      storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
81      runs = cloner.Clone(original.runs);
82      runsCounter = original.runsCounter;
83      algorithm = cloner.Clone(original.algorithm);
84      clonedAlgorithms = cloner.Clone(original.clonedAlgorithms);
85      results = cloner.Clone(original.results);
86
87      folds = cloner.Clone(original.folds);
88      numberOfWorkers = cloner.Clone(original.numberOfWorkers);
89      samplesStart = cloner.Clone(original.samplesStart);
90      samplesEnd = cloner.Clone(original.samplesEnd);
91      RegisterEvents();
92      if (Algorithm != null) RegisterAlgorithmEvents();
93    }
94    public override IDeepCloneable Clone(Cloner cloner) {
95      return new CrossValidation(this, cloner);
96    }
97
98    #endregion
99
100    #region properties
101    [Storable]
102    private IAlgorithm algorithm;
103    public IAlgorithm Algorithm {
104      get { return algorithm; }
105      set {
106        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
107          throw new InvalidOperationException("Changing the algorithm is only allowed if the CrossValidation is stopped or prepared.");
108        if (algorithm != value) {
109          if (value != null && value.Problem != null && !(value.Problem is IDataAnalysisProblem))
110            throw new ArgumentException("Only algorithms with a DataAnalysisProblem could be used for the cross validation.");
111          if (algorithm != null) DeregisterAlgorithmEvents();
112          algorithm = value;
113          Parameters.Clear();
114
115          if (algorithm != null) {
116            algorithm.StoreAlgorithmInEachRun = false;
117            RegisterAlgorithmEvents();
118            algorithm.Prepare(true);
119            Parameters.AddRange(algorithm.Parameters);
120          }
121          OnAlgorithmChanged();
122          Prepare();
123        }
124      }
125    }
126
127
128    [Storable]
129    private IDataAnalysisProblem problem;
130    public IDataAnalysisProblem Problem {
131      get {
132        if (algorithm == null)
133          return null;
134        return (IDataAnalysisProblem)algorithm.Problem;
135      }
136      set {
137        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
138          throw new InvalidOperationException("Changing the problem is only allowed if the CrossValidation is stopped or prepared.");
139        if (algorithm == null) throw new ArgumentNullException("Could not set a problem before an algorithm was set.");
140        algorithm.Problem = value;
141        problem = value;
142      }
143    }
144
145    IProblem IAlgorithm.Problem {
146      get { return Problem; }
147      set {
148        if (value != null && !ProblemType.IsInstanceOfType(value))
149          throw new ArgumentException("Only DataAnalysisProblems could be used for the cross validation.");
150        Problem = (IDataAnalysisProblem)value;
151      }
152    }
153    public Type ProblemType {
154      get { return typeof(IDataAnalysisProblem); }
155    }
156
157    [Storable]
158    private ItemCollection<IAlgorithm> clonedAlgorithms;
159
160    public IEnumerable<IOptimizer> NestedOptimizers {
161      get {
162        if (Algorithm == null) yield break;
163        yield return Algorithm;
164      }
165    }
166
167    [Storable]
168    private ResultCollection results;
169    public ResultCollection Results {
170      get { return results; }
171    }
172
173    [Storable]
174    private IntValue folds;
175    public IntValue Folds {
176      get { return folds; }
177    }
178    [Storable]
179    private IntValue samplesStart;
180    public IntValue SamplesStart {
181      get { return samplesStart; }
182    }
183    [Storable]
184    private IntValue samplesEnd;
185    public IntValue SamplesEnd {
186      get { return samplesEnd; }
187    }
188    [Storable]
189    private IntValue numberOfWorkers;
190    public IntValue NumberOfWorkers {
191      get { return numberOfWorkers; }
192    }
193
194    [Storable]
195    private bool storeAlgorithmInEachRun;
196    public bool StoreAlgorithmInEachRun {
197      get { return storeAlgorithmInEachRun; }
198      set {
199        if (storeAlgorithmInEachRun != value) {
200          storeAlgorithmInEachRun = value;
201          OnStoreAlgorithmInEachRunChanged();
202        }
203      }
204    }
205
206    [Storable]
207    private int runsCounter;
208    [Storable]
209    private RunCollection runs;
210    public RunCollection Runs {
211      get { return runs; }
212    }
213
214    [Storable]
215    private ExecutionState executionState;
216    public ExecutionState ExecutionState {
217      get { return executionState; }
218      private set {
219        if (executionState != value) {
220          executionState = value;
221          OnExecutionStateChanged();
222          OnItemImageChanged();
223        }
224      }
225    }
226    public static new Image StaticItemImage {
227      get { return HeuristicLab.Common.Resources.VSImageLibrary.Event; }
228    }
229    public override Image ItemImage {
230      get {
231        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
232        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
233        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
234        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
235        else return base.ItemImage;
236      }
237    }
238
239    public TimeSpan ExecutionTime {
240      get {
241        if (ExecutionState != ExecutionState.Prepared)
242          return TimeSpan.FromMilliseconds(clonedAlgorithms.Select(x => x.ExecutionTime.TotalMilliseconds).Sum());
243        return TimeSpan.Zero;
244      }
245    }
246    #endregion
247
248    protected override void OnNameChanged() {
249      base.OnNameChanged();
250      Runs.OptimizerName = Name;
251    }
252
253    public void Prepare() {
254      if (ExecutionState == ExecutionState.Started)
255        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
256      results.Clear();
257      clonedAlgorithms.Clear();
258      if (Algorithm != null) {
259        Algorithm.Prepare();
260        if (Algorithm.ExecutionState == ExecutionState.Prepared) OnPrepared();
261      }
262    }
263    public void Prepare(bool clearRuns) {
264      if (clearRuns) runs.Clear();
265      Prepare();
266    }
267
268    public void Start() {
269      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
270        throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));
271
272      if (Algorithm != null) {
273        //create cloned algorithms
274        if (clonedAlgorithms.Count == 0) {
275          int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / Folds.Value;
276
277          for (int i = 0; i < Folds.Value; i++) {
278            IAlgorithm clonedAlgorithm = (IAlgorithm)algorithm.Clone();
279            clonedAlgorithm.Name = algorithm.Name + " Fold " + i;
280            IDataAnalysisProblem problem = clonedAlgorithm.Problem as IDataAnalysisProblem;
281            ISymbolicDataAnalysisProblem symbolicProblem = problem as ISymbolicDataAnalysisProblem;
282
283            int testStart = (i * testSamplesCount) + SamplesStart.Value;
284            int testEnd = (i + 1) == Folds.Value ? SamplesEnd.Value : (i + 1) * testSamplesCount + SamplesStart.Value;
285
286            problem.ProblemData.TrainingPartition.Start = SamplesStart.Value;
287            problem.ProblemData.TrainingPartition.End = SamplesEnd.Value;
288            problem.ProblemData.TestPartition.Start = testStart;
289            problem.ProblemData.TestPartition.End = testEnd;
290            DataAnalysisProblemData problemData = problem.ProblemData as DataAnalysisProblemData;
291            if (problemData != null) {
292              problemData.TrainingPartitionParameter.Hidden = false;
293              problemData.TestPartitionParameter.Hidden = false;
294            }
295
296            if (symbolicProblem != null) {
297              symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
298              symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
299            }
300            clonedAlgorithm.Prepare();
301            clonedAlgorithms.Add(clonedAlgorithm);
302          }
303        }
304
305        //start prepared or paused cloned algorithms
306        int startedAlgorithms = 0;
307        foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
308          if (startedAlgorithms < NumberOfWorkers.Value) {
309            if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared ||
310                clonedAlgorithm.ExecutionState == ExecutionState.Paused) {
311
312              // start and wait until the alg is started
313              using (var signal = new ManualResetEvent(false)) {
314                EventHandler signalSetter = (sender, args) => { signal.Set(); };
315                clonedAlgorithm.Started += signalSetter;
316                clonedAlgorithm.Start();
317                signal.WaitOne();
318                clonedAlgorithm.Started -= signalSetter;
319
320                startedAlgorithms++;
321              }
322            }
323          }
324        }
325        OnStarted();
326      }
327    }
328
329    private bool pausePending;
330    public void Pause() {
331      if (ExecutionState != ExecutionState.Started)
332        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
333      if (!pausePending) {
334        pausePending = true;
335        PauseAllClonedAlgorithms();
336      }
337    }
338    private void PauseAllClonedAlgorithms() {
339      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
340        if (clonedAlgorithm.ExecutionState == ExecutionState.Started)
341          clonedAlgorithm.Pause();
342      }
343    }
344
345    private bool stopPending;
346    public void Stop() {
347      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
348        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".",
349                                                          ExecutionState));
350      if (!stopPending) {
351        stopPending = true;
352        StopAllClonedAlgorithms();
353      }
354    }
355    private void StopAllClonedAlgorithms() {
356      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
357        if (clonedAlgorithm.ExecutionState == ExecutionState.Started ||
358            clonedAlgorithm.ExecutionState == ExecutionState.Paused)
359          clonedAlgorithm.Stop();
360      }
361    }
362
363    #region collect parameters and results
364    public override void CollectParameterValues(IDictionary<string, IItem> values) {
365      values.Add("Algorithm Name", new StringValue(Name));
366      values.Add("Algorithm Type", new StringValue(GetType().GetPrettyName()));
367      values.Add("Folds", new IntValue(Folds.Value));
368
369      if (algorithm != null) {
370        values.Add("CrossValidation Algorithm Name", new StringValue(Algorithm.Name));
371        values.Add("CrossValidation Algorithm Type", new StringValue(Algorithm.GetType().GetPrettyName()));
372        base.CollectParameterValues(values);
373      }
374      if (Problem != null) {
375        values.Add("Problem Name", new StringValue(Problem.Name));
376        values.Add("Problem Type", new StringValue(Problem.GetType().GetPrettyName()));
377        Problem.CollectParameterValues(values);
378      }
379    }
380
381    public void CollectResultValues(IDictionary<string, IItem> results) {
382      var clonedResults = (ResultCollection)this.results.Clone();
383      foreach (var result in clonedResults) {
384        results.Add(result.Name, result.Value);
385      }
386    }
387
388    private void AggregateResultValues(IDictionary<string, IItem> results) {
389      IEnumerable<IRun> runs = clonedAlgorithms.Select(alg => alg.Runs.FirstOrDefault()).Where(run => run != null);
390      IEnumerable<KeyValuePair<string, IItem>> resultCollections = runs.Where(x => x != null).SelectMany(x => x.Results).ToList();
391
392      foreach (IResult result in ExtractAndAggregateResults<IntValue>(resultCollections))
393        results.Add(result.Name, result.Value);
394      foreach (IResult result in ExtractAndAggregateResults<DoubleValue>(resultCollections))
395        results.Add(result.Name, result.Value);
396      foreach (IResult result in ExtractAndAggregateResults<PercentValue>(resultCollections))
397        results.Add(result.Name, result.Value);
398      foreach (IResult result in ExtractAndAggregateRegressionSolutions(resultCollections)) {
399        results.Add(result.Name, result.Value);
400      }
401      foreach (IResult result in ExtractAndAggregateClassificationSolutions(resultCollections)) {
402        results.Add(result.Name, result.Value);
403      }
404      results.Add("Execution Time", new TimeSpanValue(this.ExecutionTime));
405      results.Add("CrossValidation Folds", new RunCollection(runs));
406    }
407
408    private IEnumerable<IResult> ExtractAndAggregateRegressionSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
409      Dictionary<string, List<IRegressionSolution>> resultSolutions = new Dictionary<string, List<IRegressionSolution>>();
410      foreach (var result in resultCollections) {
411        var regressionSolution = result.Value as IRegressionSolution;
412        if (regressionSolution != null) {
413          if (resultSolutions.ContainsKey(result.Key)) {
414            resultSolutions[result.Key].Add(regressionSolution);
415          } else {
416            resultSolutions.Add(result.Key, new List<IRegressionSolution>() { regressionSolution });
417          }
418        }
419      }
420      List<IResult> aggregatedResults = new List<IResult>();
421      foreach (KeyValuePair<string, List<IRegressionSolution>> solutions in resultSolutions) {
422        // clone manually to correctly clone references between cloned root objects
423        Cloner cloner = new Cloner();
424        var problemDataClone = (IRegressionProblemData)cloner.Clone(Problem.ProblemData);
425        // set partitions of problem data clone correctly
426        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
427        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
428        // clone models
429        var ensembleSolution = new RegressionEnsembleSolution(problemDataClone);
430        ensembleSolution.AddRegressionSolutions(solutions.Value);
431
432        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
433      }
434      List<IResult> flattenedResults = new List<IResult>();
435      CollectResultsRecursively("", aggregatedResults, flattenedResults);
436      return flattenedResults;
437    }
438
439    private IEnumerable<IResult> ExtractAndAggregateClassificationSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
440      Dictionary<string, List<IClassificationSolution>> resultSolutions = new Dictionary<string, List<IClassificationSolution>>();
441      foreach (var result in resultCollections) {
442        var classificationSolution = result.Value as IClassificationSolution;
443        if (classificationSolution != null) {
444          if (resultSolutions.ContainsKey(result.Key)) {
445            resultSolutions[result.Key].Add(classificationSolution);
446          } else {
447            resultSolutions.Add(result.Key, new List<IClassificationSolution>() { classificationSolution });
448          }
449        }
450      }
451      var aggregatedResults = new List<IResult>();
452      foreach (KeyValuePair<string, List<IClassificationSolution>> solutions in resultSolutions) {
453        // at least one algorithm (GBT with logistic regression loss) produces a classification solution even though the original problem is a regression problem.
454        var targetVariable = solutions.Value.First().ProblemData.TargetVariable;
455        var problemDataClone = new ClassificationProblemData(Problem.ProblemData.Dataset,
456          Problem.ProblemData.AllowedInputVariables, targetVariable);
457        // set partitions of problem data clone correctly
458        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
459        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
460        // clone models
461        var ensembleSolution = new ClassificationEnsembleSolution(problemDataClone);
462        ensembleSolution.AddClassificationSolutions(solutions.Value);
463
464        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
465      }
466      List<IResult> flattenedResults = new List<IResult>();
467      CollectResultsRecursively("", aggregatedResults, flattenedResults);
468      return flattenedResults;
469    }
470
471    private void CollectResultsRecursively(string path, IEnumerable<IResult> results, IList<IResult> flattenedResults) {
472      foreach (IResult result in results) {
473        flattenedResults.Add(new Result(path + result.Name, result.Value));
474        ResultCollection childCollection = result.Value as ResultCollection;
475        if (childCollection != null) {
476          CollectResultsRecursively(path + result.Name + ".", childCollection, flattenedResults);
477        }
478      }
479    }
480
481    private static IEnumerable<IResult> ExtractAndAggregateResults<T>(IEnumerable<KeyValuePair<string, IItem>> results)
482  where T : class, IItem, new() {
483      Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
484      foreach (var resultValue in results.Where(r => r.Value.GetType() == typeof(T))) {
485        if (!resultValues.ContainsKey(resultValue.Key))
486          resultValues[resultValue.Key] = new List<double>();
487        resultValues[resultValue.Key].Add(ConvertToDouble(resultValue.Value));
488      }
489
490      DoubleValue doubleValue;
491      if (typeof(T) == typeof(PercentValue))
492        doubleValue = new PercentValue();
493      else if (typeof(T) == typeof(DoubleValue))
494        doubleValue = new DoubleValue();
495      else if (typeof(T) == typeof(IntValue))
496        doubleValue = new DoubleValue();
497      else
498        throw new NotSupportedException();
499
500      List<IResult> aggregatedResults = new List<IResult>();
501      foreach (KeyValuePair<string, List<double>> resultValue in resultValues) {
502        doubleValue.Value = resultValue.Value.Average();
503        aggregatedResults.Add(new Result(resultValue.Key + " (average)", (IItem)doubleValue.Clone()));
504        doubleValue.Value = resultValue.Value.StandardDeviation();
505        aggregatedResults.Add(new Result(resultValue.Key + " (std.dev.)", (IItem)doubleValue.Clone()));
506      }
507      return aggregatedResults;
508    }
509
510    private static double ConvertToDouble(IItem item) {
511      if (item is DoubleValue) return ((DoubleValue)item).Value;
512      else if (item is IntValue) return ((IntValue)item).Value;
513      else throw new NotSupportedException("Could not convert any item type to double");
514    }
515    #endregion
516
517    #region events
518    private void RegisterEvents() {
519      Folds.ValueChanged += new EventHandler(Folds_ValueChanged);
520      RegisterClonedAlgorithmsEvents();
521    }
522    private void Folds_ValueChanged(object sender, EventArgs e) {
523      if (ExecutionState != ExecutionState.Prepared)
524        throw new InvalidOperationException("Can not change number of folds if the execution state is not prepared.");
525    }
526
527
528    #region template algorithms events
529    public event EventHandler AlgorithmChanged;
530    private void OnAlgorithmChanged() {
531      EventHandler handler = AlgorithmChanged;
532      if (handler != null) handler(this, EventArgs.Empty);
533      OnProblemChanged();
534      if (Problem == null) ExecutionState = ExecutionState.Stopped;
535    }
536    private void RegisterAlgorithmEvents() {
537      algorithm.ProblemChanged += new EventHandler(Algorithm_ProblemChanged);
538      algorithm.ExecutionStateChanged += new EventHandler(Algorithm_ExecutionStateChanged);
539      if (Problem != null) Problem.Reset += new EventHandler(Problem_Reset);
540    }
541    private void DeregisterAlgorithmEvents() {
542      algorithm.ProblemChanged -= new EventHandler(Algorithm_ProblemChanged);
543      algorithm.ExecutionStateChanged -= new EventHandler(Algorithm_ExecutionStateChanged);
544      if (Problem != null) Problem.Reset -= new EventHandler(Problem_Reset);
545    }
546    private void Algorithm_ProblemChanged(object sender, EventArgs e) {
547      if (algorithm.Problem != null && !(algorithm.Problem is IDataAnalysisProblem)) {
548        algorithm.Problem = problem;
549        throw new ArgumentException("A cross validation algorithm can only contain DataAnalysisProblems.");
550      }
551      if (problem != null) problem.Reset -= new EventHandler(Problem_Reset);
552      problem = (IDataAnalysisProblem)algorithm.Problem;
553      if (problem != null) problem.Reset += new EventHandler(Problem_Reset);
554      OnProblemChanged();
555    }
556    public event EventHandler ProblemChanged;
557    private void OnProblemChanged() {
558      EventHandler handler = ProblemChanged;
559      if (handler != null) handler(this, EventArgs.Empty);
560      ConfigureProblem();
561    }
562
563    private void Problem_Reset(object sender, EventArgs e) {
564      ConfigureProblem();
565    }
566
567    private void ConfigureProblem() {
568      SamplesStart.Value = 0;
569      if (Problem != null) {
570        SamplesEnd.Value = Problem.ProblemData.Dataset.Rows;
571
572        DataAnalysisProblemData problemData = Problem.ProblemData as DataAnalysisProblemData;
573        if (problemData != null) {
574          problemData.TrainingPartitionParameter.Hidden = true;
575          problemData.TestPartitionParameter.Hidden = true;
576        }
577        ISymbolicDataAnalysisProblem symbolicProblem = Problem as ISymbolicDataAnalysisProblem;
578        if (symbolicProblem != null) {
579          symbolicProblem.FitnessCalculationPartitionParameter.Hidden = true;
580          symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
581          symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
582          symbolicProblem.ValidationPartitionParameter.Hidden = true;
583          symbolicProblem.ValidationPartition.Start = 0;
584          symbolicProblem.ValidationPartition.End = 0;
585        }
586      } else
587        SamplesEnd.Value = 0;
588    }
589
590    private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
591      switch (Algorithm.ExecutionState) {
592        case ExecutionState.Prepared: OnPrepared();
593          break;
594        case ExecutionState.Started: throw new InvalidOperationException("Algorithm template can not be started.");
595        case ExecutionState.Paused: throw new InvalidOperationException("Algorithm template can not be paused.");
596        case ExecutionState.Stopped: OnStopped();
597          break;
598      }
599    }
600    #endregion
601
602    #region clonedAlgorithms events
603    private void RegisterClonedAlgorithmsEvents() {
604      clonedAlgorithms.ItemsAdded += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
605      clonedAlgorithms.ItemsRemoved += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
606      clonedAlgorithms.CollectionReset += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
607      foreach (IAlgorithm algorithm in clonedAlgorithms)
608        RegisterClonedAlgorithmEvents(algorithm);
609    }
610    private void DeregisterClonedAlgorithmsEvents() {
611      clonedAlgorithms.ItemsAdded -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
612      clonedAlgorithms.ItemsRemoved -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
613      clonedAlgorithms.CollectionReset -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
614      foreach (IAlgorithm algorithm in clonedAlgorithms)
615        DeregisterClonedAlgorithmEvents(algorithm);
616    }
617    private void ClonedAlgorithms_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
618      foreach (IAlgorithm algorithm in e.Items)
619        RegisterClonedAlgorithmEvents(algorithm);
620    }
621    private void ClonedAlgorithms_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
622      foreach (IAlgorithm algorithm in e.Items)
623        DeregisterClonedAlgorithmEvents(algorithm);
624    }
625    private void ClonedAlgorithms_CollectionReset(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
626      foreach (IAlgorithm algorithm in e.OldItems)
627        DeregisterClonedAlgorithmEvents(algorithm);
628      foreach (IAlgorithm algorithm in e.Items)
629        RegisterClonedAlgorithmEvents(algorithm);
630    }
631    private void RegisterClonedAlgorithmEvents(IAlgorithm algorithm) {
632      algorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
633      algorithm.ExecutionTimeChanged += new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
634      algorithm.Started += new EventHandler(ClonedAlgorithm_Started);
635      algorithm.Paused += new EventHandler(ClonedAlgorithm_Paused);
636      algorithm.Stopped += new EventHandler(ClonedAlgorithm_Stopped);
637    }
638    private void DeregisterClonedAlgorithmEvents(IAlgorithm algorithm) {
639      algorithm.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
640      algorithm.ExecutionTimeChanged -= new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
641      algorithm.Started -= new EventHandler(ClonedAlgorithm_Started);
642      algorithm.Paused -= new EventHandler(ClonedAlgorithm_Paused);
643      algorithm.Stopped -= new EventHandler(ClonedAlgorithm_Stopped);
644    }
645    private void ClonedAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
646      OnExceptionOccurred(e.Value);
647    }
648    private void ClonedAlgorithm_ExecutionTimeChanged(object sender, EventArgs e) {
649      OnExecutionTimeChanged();
650    }
651
652    private readonly object locker = new object();
653    private readonly object resultLocker = new object();
654    private void ClonedAlgorithm_Started(object sender, EventArgs e) {
655      IAlgorithm algorithm = sender as IAlgorithm;
656      lock (resultLocker) {
657        if (algorithm != null && !results.ContainsKey(algorithm.Name))
658          results.Add(new Result(algorithm.Name, "Contains results for the specific fold.", algorithm.Results));
659      }
660    }
661
662    private void ClonedAlgorithm_Paused(object sender, EventArgs e) {
663      lock (locker) {
664        if (pausePending && clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started))
665          OnPaused();
666      }
667    }
668
669    private void ClonedAlgorithm_Stopped(object sender, EventArgs e) {
670      lock (locker) {
671        if (!stopPending && ExecutionState == ExecutionState.Started) {
672          IAlgorithm preparedAlgorithm = clonedAlgorithms.FirstOrDefault(alg => alg.ExecutionState == ExecutionState.Prepared ||
673                                                                                alg.ExecutionState == ExecutionState.Paused);
674          if (preparedAlgorithm != null) preparedAlgorithm.Start();
675        }
676        if (ExecutionState != ExecutionState.Stopped) {
677          if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped))
678            OnStopped();
679          else if (stopPending &&
680                   clonedAlgorithms.All(
681                     alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped))
682            OnStopped();
683        }
684      }
685    }
686    #endregion
687    #endregion
688
689    #region event firing
690    public event EventHandler ExecutionStateChanged;
691    private void OnExecutionStateChanged() {
692      EventHandler handler = ExecutionStateChanged;
693      if (handler != null) handler(this, EventArgs.Empty);
694    }
695    public event EventHandler ExecutionTimeChanged;
696    private void OnExecutionTimeChanged() {
697      EventHandler handler = ExecutionTimeChanged;
698      if (handler != null) handler(this, EventArgs.Empty);
699    }
700    public event EventHandler Prepared;
701    private void OnPrepared() {
702      ExecutionState = ExecutionState.Prepared;
703      EventHandler handler = Prepared;
704      if (handler != null) handler(this, EventArgs.Empty);
705      OnExecutionTimeChanged();
706    }
707    public event EventHandler Started;
708    private void OnStarted() {
709      ExecutionState = ExecutionState.Started;
710      EventHandler handler = Started;
711      if (handler != null) handler(this, EventArgs.Empty);
712    }
713    public event EventHandler Paused;
714    private void OnPaused() {
715      pausePending = false;
716      ExecutionState = ExecutionState.Paused;
717      EventHandler handler = Paused;
718      if (handler != null) handler(this, EventArgs.Empty);
719    }
720    public event EventHandler Stopped;
721    private void OnStopped() {
722      stopPending = false;
723      Dictionary<string, IItem> collectedResults = new Dictionary<string, IItem>();
724      AggregateResultValues(collectedResults);
725      results.AddRange(collectedResults.Select(x => new Result(x.Key, x.Value)).Cast<IResult>().ToArray());
726      runsCounter++;
727      runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
728      ExecutionState = ExecutionState.Stopped;
729      EventHandler handler = Stopped;
730      if (handler != null) handler(this, EventArgs.Empty);
731    }
732    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
733    private void OnExceptionOccurred(Exception exception) {
734      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
735      if (handler != null) handler(this, new EventArgs<Exception>(exception));
736    }
737    public event EventHandler StoreAlgorithmInEachRunChanged;
738    private void OnStoreAlgorithmInEachRunChanged() {
739      EventHandler handler = StoreAlgorithmInEachRunChanged;
740      if (handler != null) handler(this, EventArgs.Empty);
741    }
742    #endregion
743  }
744}
Note: See TracBrowser for help on using the repository browser.