Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs @ 8929

Last change on this file since 8929 was 8836, checked in by gkronber, 12 years ago

#1534: introduced synchronization to make sure the algorithm has been started before calling OnStarted in CrossValidation

File size: 33.0 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Drawing;
25using System.Linq;
26using System.Threading;
27using HeuristicLab.Collections;
28using HeuristicLab.Common;
29using HeuristicLab.Core;
30using HeuristicLab.Data;
31using HeuristicLab.Optimization;
32using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
33using HeuristicLab.Problems.DataAnalysis;
34using HeuristicLab.Problems.DataAnalysis.Symbolic;
35
36namespace HeuristicLab.Algorithms.DataAnalysis {
37  [Item("Cross Validation", "Cross-validation wrapper for data analysis algorithms.")]
38  [Creatable("Data Analysis")]
39  [StorableClass]
40  public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent {
41    public CrossValidation()
42      : base() {
43      name = ItemName;
44      description = ItemDescription;
45
46      executionState = ExecutionState.Stopped;
47      runs = new RunCollection { AlgorithmName = name };
48      runsCounter = 0;
49
50      algorithm = null;
51      clonedAlgorithms = new ItemCollection<IAlgorithm>();
52      results = new ResultCollection();
53
54      folds = new IntValue(2);
55      numberOfWorkers = new IntValue(1);
56      samplesStart = new IntValue(0);
57      samplesEnd = new IntValue(0);
58      storeAlgorithmInEachRun = false;
59
60      RegisterEvents();
61      if (Algorithm != null) RegisterAlgorithmEvents();
62    }
63
64    public string Filename { get; set; }
65
66    #region persistence and cloning
67    [StorableConstructor]
68    private CrossValidation(bool deserializing)
69      : base(deserializing) {
70    }
71    [StorableHook(HookType.AfterDeserialization)]
72    private void AfterDeserialization() {
73      RegisterEvents();
74      if (Algorithm != null) RegisterAlgorithmEvents();
75    }
76
77    private CrossValidation(CrossValidation original, Cloner cloner)
78      : base(original, cloner) {
79      executionState = original.executionState;
80      storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
81      runs = cloner.Clone(original.runs);
82      runsCounter = original.runsCounter;
83      algorithm = cloner.Clone(original.algorithm);
84      clonedAlgorithms = cloner.Clone(original.clonedAlgorithms);
85      results = cloner.Clone(original.results);
86
87      folds = cloner.Clone(original.folds);
88      numberOfWorkers = cloner.Clone(original.numberOfWorkers);
89      samplesStart = cloner.Clone(original.samplesStart);
90      samplesEnd = cloner.Clone(original.samplesEnd);
91      RegisterEvents();
92      if (Algorithm != null) RegisterAlgorithmEvents();
93    }
94    public override IDeepCloneable Clone(Cloner cloner) {
95      return new CrossValidation(this, cloner);
96    }
97
98    #endregion
99
100    #region properties
101    [Storable]
102    private IAlgorithm algorithm;
103    public IAlgorithm Algorithm {
104      get { return algorithm; }
105      set {
106        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
107          throw new InvalidOperationException("Changing the algorithm is only allowed if the CrossValidation is stopped or prepared.");
108        if (algorithm != value) {
109          if (value != null && value.Problem != null && !(value.Problem is IDataAnalysisProblem))
110            throw new ArgumentException("Only algorithms with a DataAnalysisProblem could be used for the cross validation.");
111          if (algorithm != null) DeregisterAlgorithmEvents();
112          algorithm = value;
113          Parameters.Clear();
114
115          if (algorithm != null) {
116            algorithm.StoreAlgorithmInEachRun = false;
117            RegisterAlgorithmEvents();
118            algorithm.Prepare(true);
119            Parameters.AddRange(algorithm.Parameters);
120          }
121          OnAlgorithmChanged();
122          if (algorithm != null) OnProblemChanged();
123          Prepare();
124        }
125      }
126    }
127
128
129    [Storable]
130    private IDataAnalysisProblem problem;
131    public IDataAnalysisProblem Problem {
132      get {
133        if (algorithm == null)
134          return null;
135        return (IDataAnalysisProblem)algorithm.Problem;
136      }
137      set {
138        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
139          throw new InvalidOperationException("Changing the problem is only allowed if the CrossValidation is stopped or prepared.");
140        if (algorithm == null) throw new ArgumentNullException("Could not set a problem before an algorithm was set.");
141        algorithm.Problem = value;
142        problem = value;
143      }
144    }
145
146    IProblem IAlgorithm.Problem {
147      get { return Problem; }
148      set {
149        if (value != null && !ProblemType.IsInstanceOfType(value))
150          throw new ArgumentException("Only DataAnalysisProblems could be used for the cross validation.");
151        Problem = (IDataAnalysisProblem)value;
152      }
153    }
154    public Type ProblemType {
155      get { return typeof(IDataAnalysisProblem); }
156    }
157
158    [Storable]
159    private ItemCollection<IAlgorithm> clonedAlgorithms;
160
161    public IEnumerable<IOptimizer> NestedOptimizers {
162      get {
163        if (Algorithm == null) yield break;
164        yield return Algorithm;
165      }
166    }
167
168    [Storable]
169    private ResultCollection results;
170    public ResultCollection Results {
171      get { return results; }
172    }
173
174    [Storable]
175    private IntValue folds;
176    public IntValue Folds {
177      get { return folds; }
178    }
179    [Storable]
180    private IntValue samplesStart;
181    public IntValue SamplesStart {
182      get { return samplesStart; }
183    }
184    [Storable]
185    private IntValue samplesEnd;
186    public IntValue SamplesEnd {
187      get { return samplesEnd; }
188    }
189    [Storable]
190    private IntValue numberOfWorkers;
191    public IntValue NumberOfWorkers {
192      get { return numberOfWorkers; }
193    }
194
195    [Storable]
196    private bool storeAlgorithmInEachRun;
197    public bool StoreAlgorithmInEachRun {
198      get { return storeAlgorithmInEachRun; }
199      set {
200        if (storeAlgorithmInEachRun != value) {
201          storeAlgorithmInEachRun = value;
202          OnStoreAlgorithmInEachRunChanged();
203        }
204      }
205    }
206
207    [Storable]
208    private int runsCounter;
209    [Storable]
210    private RunCollection runs;
211    public RunCollection Runs {
212      get { return runs; }
213    }
214
215    [Storable]
216    private ExecutionState executionState;
217    public ExecutionState ExecutionState {
218      get { return executionState; }
219      private set {
220        if (executionState != value) {
221          executionState = value;
222          OnExecutionStateChanged();
223          OnItemImageChanged();
224        }
225      }
226    }
227    public static new Image StaticItemImage {
228      get { return HeuristicLab.Common.Resources.VSImageLibrary.Event; }
229    }
230    public override Image ItemImage {
231      get {
232        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
233        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
234        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
235        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
236        else return base.ItemImage;
237      }
238    }
239
240    public TimeSpan ExecutionTime {
241      get {
242        if (ExecutionState != ExecutionState.Prepared)
243          return TimeSpan.FromMilliseconds(clonedAlgorithms.Select(x => x.ExecutionTime.TotalMilliseconds).Sum());
244        return TimeSpan.Zero;
245      }
246    }
247    #endregion
248
249    protected override void OnNameChanged() {
250      base.OnNameChanged();
251      Runs.AlgorithmName = Name;
252    }
253
254    public void Prepare() {
255      if (ExecutionState == ExecutionState.Started)
256        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
257      results.Clear();
258      clonedAlgorithms.Clear();
259      if (Algorithm != null) {
260        Algorithm.Prepare();
261        if (Algorithm.ExecutionState == ExecutionState.Prepared) OnPrepared();
262      }
263    }
264    public void Prepare(bool clearRuns) {
265      if (clearRuns) runs.Clear();
266      Prepare();
267    }
268
269    private bool startPending;
270    public void Start() {
271      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
272        throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));
273
274      if (Algorithm != null && !startPending) {
275        startPending = true;
276        //create cloned algorithms
277        if (clonedAlgorithms.Count == 0) {
278          int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / Folds.Value;
279
280          for (int i = 0; i < Folds.Value; i++) {
281            IAlgorithm clonedAlgorithm = (IAlgorithm)algorithm.Clone();
282            clonedAlgorithm.Name = algorithm.Name + " Fold " + i;
283            IDataAnalysisProblem problem = clonedAlgorithm.Problem as IDataAnalysisProblem;
284            ISymbolicDataAnalysisProblem symbolicProblem = problem as ISymbolicDataAnalysisProblem;
285
286            int testStart = (i * testSamplesCount) + SamplesStart.Value;
287            int testEnd = (i + 1) == Folds.Value ? SamplesEnd.Value : (i + 1) * testSamplesCount + SamplesStart.Value;
288
289            problem.ProblemData.TrainingPartition.Start = SamplesStart.Value;
290            problem.ProblemData.TrainingPartition.End = SamplesEnd.Value;
291            problem.ProblemData.TestPartition.Start = testStart;
292            problem.ProblemData.TestPartition.End = testEnd;
293            DataAnalysisProblemData problemData = problem.ProblemData as DataAnalysisProblemData;
294            if (problemData != null) {
295              problemData.TrainingPartitionParameter.Hidden = false;
296              problemData.TestPartitionParameter.Hidden = false;
297            }
298
299            if (symbolicProblem != null) {
300              symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
301              symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
302            }
303            clonedAlgorithm.Prepare();
304            clonedAlgorithms.Add(clonedAlgorithm);
305          }
306        }
307
308        //start prepared or paused cloned algorithms
309        int startedAlgorithms = 0;
310        foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
311          if (startedAlgorithms < NumberOfWorkers.Value) {
312            if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared ||
313                clonedAlgorithm.ExecutionState == ExecutionState.Paused) {
314
315              // start and wait until the alg is started
316              using (var signal = new ManualResetEvent(false)) {
317                EventHandler signalSetter = (sender, args) => { signal.Set(); };
318                clonedAlgorithm.Started += signalSetter;
319                clonedAlgorithm.Start();
320                signal.WaitOne();
321                clonedAlgorithm.Started -= signalSetter;
322
323                startedAlgorithms++;
324              }
325            }
326          }
327        }
328        OnStarted();
329      }
330    }
331
332    private bool pausePending;
333    public void Pause() {
334      if (ExecutionState != ExecutionState.Started)
335        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
336      if (!pausePending) {
337        pausePending = true;
338        if (!startPending) PauseAllClonedAlgorithms();
339      }
340    }
341    private void PauseAllClonedAlgorithms() {
342      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
343        if (clonedAlgorithm.ExecutionState == ExecutionState.Started)
344          clonedAlgorithm.Pause();
345      }
346    }
347
348    private bool stopPending;
349    public void Stop() {
350      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
351        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".",
352                                                          ExecutionState));
353      if (!stopPending) {
354        stopPending = true;
355        if (!startPending) StopAllClonedAlgorithms();
356      }
357    }
358    private void StopAllClonedAlgorithms() {
359      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
360        if (clonedAlgorithm.ExecutionState == ExecutionState.Started ||
361            clonedAlgorithm.ExecutionState == ExecutionState.Paused)
362          clonedAlgorithm.Stop();
363      }
364    }
365
366    #region collect parameters and results
367    public override void CollectParameterValues(IDictionary<string, IItem> values) {
368      values.Add("Algorithm Name", new StringValue(Name));
369      values.Add("Algorithm Type", new StringValue(GetType().GetPrettyName()));
370      values.Add("Folds", new IntValue(Folds.Value));
371
372      if (algorithm != null) {
373        values.Add("CrossValidation Algorithm Name", new StringValue(Algorithm.Name));
374        values.Add("CrossValidation Algorithm Type", new StringValue(Algorithm.GetType().GetPrettyName()));
375        base.CollectParameterValues(values);
376      }
377      if (Problem != null) {
378        values.Add("Problem Name", new StringValue(Problem.Name));
379        values.Add("Problem Type", new StringValue(Problem.GetType().GetPrettyName()));
380        Problem.CollectParameterValues(values);
381      }
382    }
383
384    public void CollectResultValues(IDictionary<string, IItem> results) {
385      var clonedResults = (ResultCollection)this.results.Clone();
386      foreach (var result in clonedResults) {
387        results.Add(result.Name, result.Value);
388      }
389    }
390
391    private void AggregateResultValues(IDictionary<string, IItem> results) {
392      Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
393      IEnumerable<IRun> runs = clonedAlgorithms.Select(alg => alg.Runs.FirstOrDefault()).Where(run => run != null);
394      IEnumerable<KeyValuePair<string, IItem>> resultCollections = runs.Where(x => x != null).SelectMany(x => x.Results).ToList();
395
396      foreach (IResult result in ExtractAndAggregateResults<IntValue>(resultCollections))
397        results.Add(result.Name, result.Value);
398      foreach (IResult result in ExtractAndAggregateResults<DoubleValue>(resultCollections))
399        results.Add(result.Name, result.Value);
400      foreach (IResult result in ExtractAndAggregateResults<PercentValue>(resultCollections))
401        results.Add(result.Name, result.Value);
402      foreach (IResult result in ExtractAndAggregateRegressionSolutions(resultCollections)) {
403        results.Add(result.Name, result.Value);
404      }
405      foreach (IResult result in ExtractAndAggregateClassificationSolutions(resultCollections)) {
406        results.Add(result.Name, result.Value);
407      }
408      results.Add("Execution Time", new TimeSpanValue(this.ExecutionTime));
409      results.Add("CrossValidation Folds", new RunCollection(runs));
410    }
411
412    private IEnumerable<IResult> ExtractAndAggregateRegressionSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
413      Dictionary<string, List<IRegressionSolution>> resultSolutions = new Dictionary<string, List<IRegressionSolution>>();
414      foreach (var result in resultCollections) {
415        var regressionSolution = result.Value as IRegressionSolution;
416        if (regressionSolution != null) {
417          if (resultSolutions.ContainsKey(result.Key)) {
418            resultSolutions[result.Key].Add(regressionSolution);
419          } else {
420            resultSolutions.Add(result.Key, new List<IRegressionSolution>() { regressionSolution });
421          }
422        }
423      }
424      List<IResult> aggregatedResults = new List<IResult>();
425      foreach (KeyValuePair<string, List<IRegressionSolution>> solutions in resultSolutions) {
426        // clone manually to correctly clone references between cloned root objects
427        Cloner cloner = new Cloner();
428        var problemDataClone = (IRegressionProblemData)cloner.Clone(Problem.ProblemData);
429        // set partitions of problem data clone correctly
430        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
431        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
432        // clone models
433        var ensembleSolution = new RegressionEnsembleSolution(problemDataClone);
434        ensembleSolution.AddRegressionSolutions(solutions.Value);
435
436        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
437      }
438      List<IResult> flattenedResults = new List<IResult>();
439      CollectResultsRecursively("", aggregatedResults, flattenedResults);
440      return flattenedResults;
441    }
442
443    private IEnumerable<IResult> ExtractAndAggregateClassificationSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
444      Dictionary<string, List<IClassificationSolution>> resultSolutions = new Dictionary<string, List<IClassificationSolution>>();
445      foreach (var result in resultCollections) {
446        var classificationSolution = result.Value as IClassificationSolution;
447        if (classificationSolution != null) {
448          if (resultSolutions.ContainsKey(result.Key)) {
449            resultSolutions[result.Key].Add(classificationSolution);
450          } else {
451            resultSolutions.Add(result.Key, new List<IClassificationSolution>() { classificationSolution });
452          }
453        }
454      }
455      var aggregatedResults = new List<IResult>();
456      foreach (KeyValuePair<string, List<IClassificationSolution>> solutions in resultSolutions) {
457        // clone manually to correctly clone references between cloned root objects
458        Cloner cloner = new Cloner();
459        var problemDataClone = (IClassificationProblemData)cloner.Clone(Problem.ProblemData);
460        // set partitions of problem data clone correctly
461        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
462        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
463        // clone models
464        var ensembleSolution = new ClassificationEnsembleSolution(problemDataClone);
465        ensembleSolution.AddClassificationSolutions(solutions.Value);
466
467        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
468      }
469      List<IResult> flattenedResults = new List<IResult>();
470      CollectResultsRecursively("", aggregatedResults, flattenedResults);
471      return flattenedResults;
472    }
473
474    private void CollectResultsRecursively(string path, IEnumerable<IResult> results, IList<IResult> flattenedResults) {
475      foreach (IResult result in results) {
476        flattenedResults.Add(new Result(path + result.Name, result.Value));
477        ResultCollection childCollection = result.Value as ResultCollection;
478        if (childCollection != null) {
479          CollectResultsRecursively(path + result.Name + ".", childCollection, flattenedResults);
480        }
481      }
482    }
483
484    private static IEnumerable<IResult> ExtractAndAggregateResults<T>(IEnumerable<KeyValuePair<string, IItem>> results)
485  where T : class, IItem, new() {
486      Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
487      foreach (var resultValue in results.Where(r => r.Value.GetType() == typeof(T))) {
488        if (!resultValues.ContainsKey(resultValue.Key))
489          resultValues[resultValue.Key] = new List<double>();
490        resultValues[resultValue.Key].Add(ConvertToDouble(resultValue.Value));
491      }
492
493      DoubleValue doubleValue;
494      if (typeof(T) == typeof(PercentValue))
495        doubleValue = new PercentValue();
496      else if (typeof(T) == typeof(DoubleValue))
497        doubleValue = new DoubleValue();
498      else if (typeof(T) == typeof(IntValue))
499        doubleValue = new DoubleValue();
500      else
501        throw new NotSupportedException();
502
503      List<IResult> aggregatedResults = new List<IResult>();
504      foreach (KeyValuePair<string, List<double>> resultValue in resultValues) {
505        doubleValue.Value = resultValue.Value.Average();
506        aggregatedResults.Add(new Result(resultValue.Key + " (average)", (IItem)doubleValue.Clone()));
507        doubleValue.Value = resultValue.Value.StandardDeviation();
508        aggregatedResults.Add(new Result(resultValue.Key + " (std.dev.)", (IItem)doubleValue.Clone()));
509      }
510      return aggregatedResults;
511    }
512
513    private static double ConvertToDouble(IItem item) {
514      if (item is DoubleValue) return ((DoubleValue)item).Value;
515      else if (item is IntValue) return ((IntValue)item).Value;
516      else throw new NotSupportedException("Could not convert any item type to double");
517    }
518    #endregion
519
520    #region events
521    private void RegisterEvents() {
522      Folds.ValueChanged += new EventHandler(Folds_ValueChanged);
523      SamplesStart.ValueChanged += new EventHandler(SamplesStart_ValueChanged);
524      SamplesEnd.ValueChanged += new EventHandler(SamplesEnd_ValueChanged);
525      RegisterClonedAlgorithmsEvents();
526    }
527    private void Folds_ValueChanged(object sender, EventArgs e) {
528      if (ExecutionState != ExecutionState.Prepared)
529        throw new InvalidOperationException("Can not change number of folds if the execution state is not prepared.");
530    }
531
532    private bool samplesChanged = false;
533    private void SamplesStart_ValueChanged(object sender, EventArgs e) {
534      samplesChanged = true;
535      if (Problem != null) Problem.ProblemData.TrainingPartition.Start = SamplesStart.Value;
536      samplesChanged = false;
537    }
538    private void SamplesEnd_ValueChanged(object sender, EventArgs e) {
539      samplesChanged = true;
540      if (Problem != null) Problem.ProblemData.TrainingPartition.End = SamplesEnd.Value;
541      samplesChanged = false;
542    }
543
544    #region template algorithms events
545    public event EventHandler AlgorithmChanged;
546    private void OnAlgorithmChanged() {
547      EventHandler handler = AlgorithmChanged;
548      if (handler != null) handler(this, EventArgs.Empty);
549      OnProblemChanged();
550      if (Problem == null) ExecutionState = ExecutionState.Stopped;
551    }
552    private void RegisterAlgorithmEvents() {
553      algorithm.ProblemChanged += new EventHandler(Algorithm_ProblemChanged);
554      algorithm.ExecutionStateChanged += new EventHandler(Algorithm_ExecutionStateChanged);
555    }
556    private void DeregisterAlgorithmEvents() {
557      algorithm.ProblemChanged -= new EventHandler(Algorithm_ProblemChanged);
558      algorithm.ExecutionStateChanged -= new EventHandler(Algorithm_ExecutionStateChanged);
559    }
560    private void Algorithm_ProblemChanged(object sender, EventArgs e) {
561      if (algorithm.Problem != null && !(algorithm.Problem is IDataAnalysisProblem)) {
562        algorithm.Problem = problem;
563        throw new ArgumentException("A cross validation algorithm can only contain DataAnalysisProblems.");
564      }
565      algorithm.Problem.Reset += (x, y) => OnProblemChanged();
566      problem = (IDataAnalysisProblem)algorithm.Problem;
567      OnProblemChanged();
568    }
569    public event EventHandler ProblemChanged;
570    private void OnProblemChanged() {
571      EventHandler handler = ProblemChanged;
572      if (handler != null) handler(this, EventArgs.Empty);
573      if (samplesChanged) return;
574
575      SamplesStart.Value = 0;
576      if (Problem != null) {
577        SamplesEnd.Value = Problem.ProblemData.Dataset.Rows;
578
579        DataAnalysisProblemData problemData = Problem.ProblemData as DataAnalysisProblemData;
580        if (problemData != null) {
581          problemData.TrainingPartitionParameter.Hidden = true;
582          problemData.TestPartitionParameter.Hidden = true;
583        }
584        ISymbolicDataAnalysisProblem symbolicProblem = Problem as ISymbolicDataAnalysisProblem;
585        if (symbolicProblem != null) {
586          symbolicProblem.FitnessCalculationPartitionParameter.Hidden = true;
587          symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
588          symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
589          symbolicProblem.ValidationPartitionParameter.Hidden = true;
590          symbolicProblem.ValidationPartition.Start = 0;
591          symbolicProblem.ValidationPartition.End = 0;
592        }
593      } else
594        SamplesEnd.Value = 0;
595
596      SamplesStart_ValueChanged(this, EventArgs.Empty);
597      SamplesEnd_ValueChanged(this, EventArgs.Empty);
598    }
599
600    private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
601      switch (Algorithm.ExecutionState) {
602        case ExecutionState.Prepared: OnPrepared();
603          break;
604        case ExecutionState.Started: throw new InvalidOperationException("Algorithm template can not be started.");
605        case ExecutionState.Paused: throw new InvalidOperationException("Algorithm template can not be paused.");
606        case ExecutionState.Stopped: OnStopped();
607          break;
608      }
609    }
610    #endregion
611
612    #region clonedAlgorithms events
613    private void RegisterClonedAlgorithmsEvents() {
614      clonedAlgorithms.ItemsAdded += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
615      clonedAlgorithms.ItemsRemoved += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
616      clonedAlgorithms.CollectionReset += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
617      foreach (IAlgorithm algorithm in clonedAlgorithms)
618        RegisterClonedAlgorithmEvents(algorithm);
619    }
620    private void DeregisterClonedAlgorithmsEvents() {
621      clonedAlgorithms.ItemsAdded -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
622      clonedAlgorithms.ItemsRemoved -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
623      clonedAlgorithms.CollectionReset -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
624      foreach (IAlgorithm algorithm in clonedAlgorithms)
625        DeregisterClonedAlgorithmEvents(algorithm);
626    }
627    private void ClonedAlgorithms_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
628      foreach (IAlgorithm algorithm in e.Items)
629        RegisterClonedAlgorithmEvents(algorithm);
630    }
631    private void ClonedAlgorithms_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
632      foreach (IAlgorithm algorithm in e.Items)
633        DeregisterClonedAlgorithmEvents(algorithm);
634    }
635    private void ClonedAlgorithms_CollectionReset(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
636      foreach (IAlgorithm algorithm in e.OldItems)
637        DeregisterClonedAlgorithmEvents(algorithm);
638      foreach (IAlgorithm algorithm in e.Items)
639        RegisterClonedAlgorithmEvents(algorithm);
640    }
641    private void RegisterClonedAlgorithmEvents(IAlgorithm algorithm) {
642      algorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
643      algorithm.ExecutionTimeChanged += new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
644      algorithm.Started += new EventHandler(ClonedAlgorithm_Started);
645      algorithm.Paused += new EventHandler(ClonedAlgorithm_Paused);
646      algorithm.Stopped += new EventHandler(ClonedAlgorithm_Stopped);
647    }
648    private void DeregisterClonedAlgorithmEvents(IAlgorithm algorithm) {
649      algorithm.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
650      algorithm.ExecutionTimeChanged -= new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
651      algorithm.Started -= new EventHandler(ClonedAlgorithm_Started);
652      algorithm.Paused -= new EventHandler(ClonedAlgorithm_Paused);
653      algorithm.Stopped -= new EventHandler(ClonedAlgorithm_Stopped);
654    }
655    private void ClonedAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
656      OnExceptionOccurred(e.Value);
657    }
658    private void ClonedAlgorithm_ExecutionTimeChanged(object sender, EventArgs e) {
659      OnExecutionTimeChanged();
660    }
661
662    private readonly object locker = new object();
663    private void ClonedAlgorithm_Started(object sender, EventArgs e) {
664      lock (locker) {
665        IAlgorithm algorithm = sender as IAlgorithm;
666        if (algorithm != null && !results.ContainsKey(algorithm.Name))
667          results.Add(new Result(algorithm.Name, "Contains results for the specific fold.", algorithm.Results));
668
669        if (startPending) {
670          int startedAlgorithms = clonedAlgorithms.Count(alg => alg.ExecutionState == ExecutionState.Started);
671          if (startedAlgorithms == NumberOfWorkers.Value ||
672             clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Prepared))
673            startPending = false;
674
675          if (pausePending) PauseAllClonedAlgorithms();
676          if (stopPending) StopAllClonedAlgorithms();
677        }
678      }
679    }
680
681    private void ClonedAlgorithm_Paused(object sender, EventArgs e) {
682      lock (locker) {
683        if (pausePending && clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started))
684          OnPaused();
685      }
686    }
687
688    private void ClonedAlgorithm_Stopped(object sender, EventArgs e) {
689      lock (locker) {
690        if (!stopPending && ExecutionState == ExecutionState.Started) {
691          IAlgorithm preparedAlgorithm = clonedAlgorithms.Where(alg => alg.ExecutionState == ExecutionState.Prepared ||
692                                                                       alg.ExecutionState == ExecutionState.Paused).FirstOrDefault();
693          if (preparedAlgorithm != null) preparedAlgorithm.Start();
694        }
695        if (ExecutionState != ExecutionState.Stopped) {
696          if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped))
697            OnStopped();
698          else if (stopPending &&
699                   clonedAlgorithms.All(
700                     alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped))
701            OnStopped();
702        }
703      }
704    }
705    #endregion
706    #endregion
707
708    #region event firing
709    public event EventHandler ExecutionStateChanged;
710    private void OnExecutionStateChanged() {
711      EventHandler handler = ExecutionStateChanged;
712      if (handler != null) handler(this, EventArgs.Empty);
713    }
714    public event EventHandler ExecutionTimeChanged;
715    private void OnExecutionTimeChanged() {
716      EventHandler handler = ExecutionTimeChanged;
717      if (handler != null) handler(this, EventArgs.Empty);
718    }
719    public event EventHandler Prepared;
720    private void OnPrepared() {
721      ExecutionState = ExecutionState.Prepared;
722      EventHandler handler = Prepared;
723      if (handler != null) handler(this, EventArgs.Empty);
724      OnExecutionTimeChanged();
725    }
726    public event EventHandler Started;
727    private void OnStarted() {
728      startPending = false;
729      ExecutionState = ExecutionState.Started;
730      EventHandler handler = Started;
731      if (handler != null) handler(this, EventArgs.Empty);
732    }
733    public event EventHandler Paused;
734    private void OnPaused() {
735      pausePending = false;
736      ExecutionState = ExecutionState.Paused;
737      EventHandler handler = Paused;
738      if (handler != null) handler(this, EventArgs.Empty);
739    }
740    public event EventHandler Stopped;
741    private void OnStopped() {
742      stopPending = false;
743      Dictionary<string, IItem> collectedResults = new Dictionary<string, IItem>();
744      AggregateResultValues(collectedResults);
745      results.AddRange(collectedResults.Select(x => new Result(x.Key, x.Value)).Cast<IResult>().ToArray());
746      runsCounter++;
747      runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
748      ExecutionState = ExecutionState.Stopped;
749      EventHandler handler = Stopped;
750      if (handler != null) handler(this, EventArgs.Empty);
751    }
752    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
753    private void OnExceptionOccurred(Exception exception) {
754      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
755      if (handler != null) handler(this, new EventArgs<Exception>(exception));
756    }
757    public event EventHandler StoreAlgorithmInEachRunChanged;
758    private void OnStoreAlgorithmInEachRunChanged() {
759      EventHandler handler = StoreAlgorithmInEachRunChanged;
760      if (handler != null) handler(this, EventArgs.Empty);
761    }
762    #endregion
763  }
764}
Note: See TracBrowser for help on using the repository browser.