Free cookie consent management tool by TermsFeed Policy Generator

source: branches/RegressionBenchmarks/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs @ 7290

Last change on this file since 7290 was 7290, checked in by gkronber, 12 years ago

#1669 merged r7209:7283 from trunk into regression benchmark branch

File size: 32.8 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Drawing;
25using System.Linq;
26using HeuristicLab.Collections;
27using HeuristicLab.Common;
28using HeuristicLab.Core;
29using HeuristicLab.Data;
30using HeuristicLab.Optimization;
31using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
32using HeuristicLab.Problems.DataAnalysis;
33using HeuristicLab.Problems.DataAnalysis.Symbolic;
34
35namespace HeuristicLab.Algorithms.DataAnalysis {
36  [Item("Cross Validation", "Cross-validation wrapper for data analysis algorithms.")]
37  [Creatable("Data Analysis")]
38  [StorableClass]
39  public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent {
40    public CrossValidation()
41      : base() {
42      name = ItemName;
43      description = ItemDescription;
44
45      executionState = ExecutionState.Stopped;
46      runs = new RunCollection();
47      runsCounter = 0;
48
49      algorithm = null;
50      clonedAlgorithms = new ItemCollection<IAlgorithm>();
51      results = new ResultCollection();
52
53      folds = new IntValue(2);
54      numberOfWorkers = new IntValue(1);
55      samplesStart = new IntValue(0);
56      samplesEnd = new IntValue(0);
57      storeAlgorithmInEachRun = false;
58
59      RegisterEvents();
60      if (Algorithm != null) RegisterAlgorithmEvents();
61    }
62
63    public string Filename { get; set; }
64
65    #region persistence and cloning
66    [StorableConstructor]
67    private CrossValidation(bool deserializing)
68      : base(deserializing) {
69    }
70    [StorableHook(HookType.AfterDeserialization)]
71    private void AfterDeserialization() {
72      RegisterEvents();
73      if (Algorithm != null) RegisterAlgorithmEvents();
74    }
75
76    private CrossValidation(CrossValidation original, Cloner cloner)
77      : base(original, cloner) {
78      executionState = original.executionState;
79      storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
80      runs = cloner.Clone(original.runs);
81      runsCounter = original.runsCounter;
82      algorithm = cloner.Clone(original.algorithm);
83      clonedAlgorithms = cloner.Clone(original.clonedAlgorithms);
84      results = cloner.Clone(original.results);
85
86      folds = cloner.Clone(original.folds);
87      numberOfWorkers = cloner.Clone(original.numberOfWorkers);
88      samplesStart = cloner.Clone(original.samplesStart);
89      samplesEnd = cloner.Clone(original.samplesEnd);
90      RegisterEvents();
91      if (Algorithm != null) RegisterAlgorithmEvents();
92    }
93    public override IDeepCloneable Clone(Cloner cloner) {
94      return new CrossValidation(this, cloner);
95    }
96
97    #endregion
98
99    #region properties
100    [Storable]
101    private IAlgorithm algorithm;
102    public IAlgorithm Algorithm {
103      get { return algorithm; }
104      set {
105        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
106          throw new InvalidOperationException("Changing the algorithm is only allowed if the CrossValidation is stopped or prepared.");
107        if (algorithm != value) {
108          if (value != null && value.Problem != null && !(value.Problem is IDataAnalysisProblem))
109            throw new ArgumentException("Only algorithms with a DataAnalysisProblem could be used for the cross validation.");
110          if (algorithm != null) DeregisterAlgorithmEvents();
111          algorithm = value;
112          Parameters.Clear();
113
114          if (algorithm != null) {
115            algorithm.StoreAlgorithmInEachRun = false;
116            RegisterAlgorithmEvents();
117            algorithm.Prepare(true);
118            Parameters.AddRange(algorithm.Parameters);
119          }
120          OnAlgorithmChanged();
121          if (algorithm != null) OnProblemChanged();
122          Prepare();
123        }
124      }
125    }
126
127
128    [Storable]
129    private IDataAnalysisProblem problem;
130    public IDataAnalysisProblem Problem {
131      get {
132        if (algorithm == null)
133          return null;
134        return (IDataAnalysisProblem)algorithm.Problem;
135      }
136      set {
137        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
138          throw new InvalidOperationException("Changing the problem is only allowed if the CrossValidation is stopped or prepared.");
139        if (algorithm == null) throw new ArgumentNullException("Could not set a problem before an algorithm was set.");
140        algorithm.Problem = value;
141        problem = value;
142      }
143    }
144
145    IProblem IAlgorithm.Problem {
146      get { return Problem; }
147      set {
148        if (value != null && !ProblemType.IsInstanceOfType(value))
149          throw new ArgumentException("Only DataAnalysisProblems could be used for the cross validation.");
150        Problem = (IDataAnalysisProblem)value;
151      }
152    }
153    public Type ProblemType {
154      get { return typeof(IDataAnalysisProblem); }
155    }
156
157    [Storable]
158    private ItemCollection<IAlgorithm> clonedAlgorithms;
159
160    public IEnumerable<IOptimizer> NestedOptimizers {
161      get {
162        if (Algorithm == null) yield break;
163        yield return Algorithm;
164      }
165    }
166
167    [Storable]
168    private ResultCollection results;
169    public ResultCollection Results {
170      get { return results; }
171    }
172
173    [Storable]
174    private IntValue folds;
175    public IntValue Folds {
176      get { return folds; }
177    }
178    [Storable]
179    private IntValue samplesStart;
180    public IntValue SamplesStart {
181      get { return samplesStart; }
182    }
183    [Storable]
184    private IntValue samplesEnd;
185    public IntValue SamplesEnd {
186      get { return samplesEnd; }
187    }
188    [Storable]
189    private IntValue numberOfWorkers;
190    public IntValue NumberOfWorkers {
191      get { return numberOfWorkers; }
192    }
193
194    [Storable]
195    private bool storeAlgorithmInEachRun;
196    public bool StoreAlgorithmInEachRun {
197      get { return storeAlgorithmInEachRun; }
198      set {
199        if (storeAlgorithmInEachRun != value) {
200          storeAlgorithmInEachRun = value;
201          OnStoreAlgorithmInEachRunChanged();
202        }
203      }
204    }
205
206    [Storable]
207    private int runsCounter;
208    [Storable]
209    private RunCollection runs;
210    public RunCollection Runs {
211      get { return runs; }
212    }
213
214    [Storable]
215    private ExecutionState executionState;
216    public ExecutionState ExecutionState {
217      get { return executionState; }
218      private set {
219        if (executionState != value) {
220          executionState = value;
221          OnExecutionStateChanged();
222          OnItemImageChanged();
223        }
224      }
225    }
226    public static new Image StaticItemImage {
227      get { return HeuristicLab.Common.Resources.VSImageLibrary.Event; }
228    }
229    public override Image ItemImage {
230      get {
231        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
232        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
233        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
234        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
235        else return base.ItemImage;
236      }
237    }
238
239    public TimeSpan ExecutionTime {
240      get {
241        if (ExecutionState != ExecutionState.Prepared)
242          return TimeSpan.FromMilliseconds(clonedAlgorithms.Select(x => x.ExecutionTime.TotalMilliseconds).Sum());
243        return TimeSpan.Zero;
244      }
245    }
246    #endregion
247
248    public void Prepare() {
249      if (ExecutionState == ExecutionState.Started)
250        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
251      results.Clear();
252      clonedAlgorithms.Clear();
253      if (Algorithm != null) {
254        Algorithm.Prepare();
255        if (Algorithm.ExecutionState == ExecutionState.Prepared) OnPrepared();
256      }
257    }
258    public void Prepare(bool clearRuns) {
259      if (clearRuns) runs.Clear();
260      Prepare();
261    }
262
263    private bool startPending;
264    public void Start() {
265      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
266        throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));
267
268      if (Algorithm != null && !startPending) {
269        startPending = true;
270        //create cloned algorithms
271        if (clonedAlgorithms.Count == 0) {
272          int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / Folds.Value;
273
274          for (int i = 0; i < Folds.Value; i++) {
275            IAlgorithm clonedAlgorithm = (IAlgorithm)algorithm.Clone();
276            clonedAlgorithm.Name = algorithm.Name + " Fold " + i;
277            IDataAnalysisProblem problem = clonedAlgorithm.Problem as IDataAnalysisProblem;
278            ISymbolicDataAnalysisProblem symbolicProblem = problem as ISymbolicDataAnalysisProblem;
279
280            int testStart = (i * testSamplesCount) + SamplesStart.Value;
281            int testEnd = (i + 1) == Folds.Value ? SamplesEnd.Value : (i + 1) * testSamplesCount + SamplesStart.Value;
282
283            problem.ProblemData.TrainingPartition.Start = SamplesStart.Value;
284            problem.ProblemData.TrainingPartition.End = SamplesEnd.Value;
285            problem.ProblemData.TestPartition.Start = testStart;
286            problem.ProblemData.TestPartition.End = testEnd;
287            DataAnalysisProblemData problemData = problem.ProblemData as DataAnalysisProblemData;
288            if (problemData != null) {
289              problemData.TrainingPartitionParameter.Hidden = false;
290              problemData.TestPartitionParameter.Hidden = false;
291            }
292
293            if (symbolicProblem != null) {
294              symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
295              symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
296            }
297
298            clonedAlgorithms.Add(clonedAlgorithm);
299          }
300        }
301
302        //start prepared or paused cloned algorithms
303        int startedAlgorithms = 0;
304        foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
305          if (startedAlgorithms < NumberOfWorkers.Value) {
306            if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared ||
307                clonedAlgorithm.ExecutionState == ExecutionState.Paused) {
308              clonedAlgorithm.Start();
309              startedAlgorithms++;
310            }
311          }
312        }
313        OnStarted();
314      }
315    }
316
317    private bool pausePending;
318    public void Pause() {
319      if (ExecutionState != ExecutionState.Started)
320        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
321      if (!pausePending) {
322        pausePending = true;
323        if (!startPending) PauseAllClonedAlgorithms();
324      }
325    }
326    private void PauseAllClonedAlgorithms() {
327      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
328        if (clonedAlgorithm.ExecutionState == ExecutionState.Started)
329          clonedAlgorithm.Pause();
330      }
331    }
332
333    private bool stopPending;
334    public void Stop() {
335      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
336        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".",
337                                                          ExecutionState));
338      if (!stopPending) {
339        stopPending = true;
340        if (!startPending) StopAllClonedAlgorithms();
341      }
342    }
343    private void StopAllClonedAlgorithms() {
344      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
345        if (clonedAlgorithm.ExecutionState == ExecutionState.Started ||
346            clonedAlgorithm.ExecutionState == ExecutionState.Paused)
347          clonedAlgorithm.Stop();
348      }
349    }
350
351    #region collect parameters and results
352    public override void CollectParameterValues(IDictionary<string, IItem> values) {
353      values.Add("Algorithm Name", new StringValue(Name));
354      values.Add("Algorithm Type", new StringValue(GetType().GetPrettyName()));
355      values.Add("Folds", new IntValue(Folds.Value));
356
357      if (algorithm != null) {
358        values.Add("CrossValidation Algorithm Name", new StringValue(Algorithm.Name));
359        values.Add("CrossValidation Algorithm Type", new StringValue(Algorithm.GetType().GetPrettyName()));
360        base.CollectParameterValues(values);
361      }
362      if (Problem != null) {
363        values.Add("Problem Name", new StringValue(Problem.Name));
364        values.Add("Problem Type", new StringValue(Problem.GetType().GetPrettyName()));
365        Problem.CollectParameterValues(values);
366      }
367    }
368
369    public void CollectResultValues(IDictionary<string, IItem> results) {
370      var clonedResults = (ResultCollection)this.results.Clone();
371      foreach (var result in clonedResults) {
372        results.Add(result.Name, result.Value);
373      }
374    }
375
376    private void AggregateResultValues(IDictionary<string, IItem> results) {
377      Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
378      IEnumerable<IRun> runs = clonedAlgorithms.Select(alg => alg.Runs.FirstOrDefault()).Where(run => run != null);
379      IEnumerable<KeyValuePair<string, IItem>> resultCollections = runs.Where(x => x != null).SelectMany(x => x.Results).ToList();
380
381      foreach (IResult result in ExtractAndAggregateResults<IntValue>(resultCollections))
382        results.Add(result.Name, result.Value);
383      foreach (IResult result in ExtractAndAggregateResults<DoubleValue>(resultCollections))
384        results.Add(result.Name, result.Value);
385      foreach (IResult result in ExtractAndAggregateResults<PercentValue>(resultCollections))
386        results.Add(result.Name, result.Value);
387      foreach (IResult result in ExtractAndAggregateRegressionSolutions(resultCollections)) {
388        results.Add(result.Name, result.Value);
389      }
390      foreach (IResult result in ExtractAndAggregateClassificationSolutions(resultCollections)) {
391        results.Add(result.Name, result.Value);
392      }
393      results.Add("Execution Time", new TimeSpanValue(this.ExecutionTime));
394      results.Add("CrossValidation Folds", new RunCollection(runs));
395    }
396
397    private IEnumerable<IResult> ExtractAndAggregateRegressionSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
398      Dictionary<string, List<IRegressionSolution>> resultSolutions = new Dictionary<string, List<IRegressionSolution>>();
399      foreach (var result in resultCollections) {
400        var regressionSolution = result.Value as IRegressionSolution;
401        if (regressionSolution != null) {
402          if (resultSolutions.ContainsKey(result.Key)) {
403            resultSolutions[result.Key].Add(regressionSolution);
404          } else {
405            resultSolutions.Add(result.Key, new List<IRegressionSolution>() { regressionSolution });
406          }
407        }
408      }
409      List<IResult> aggregatedResults = new List<IResult>();
410      foreach (KeyValuePair<string, List<IRegressionSolution>> solutions in resultSolutions) {
411        // clone manually to correctly clone references between cloned root objects
412        Cloner cloner = new Cloner();
413        var problemDataClone = (IRegressionProblemData)cloner.Clone(Problem.ProblemData);
414        // set partitions of problem data clone correctly
415        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
416        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
417        // clone models
418        var ensembleSolution = new RegressionEnsembleSolution(
419          solutions.Value.Select(x => cloner.Clone(x.Model)),
420          problemDataClone,
421          solutions.Value.Select(x => cloner.Clone(x.ProblemData.TrainingPartition)),
422          solutions.Value.Select(x => cloner.Clone(x.ProblemData.TestPartition)));
423
424        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
425      }
426      List<IResult> flattenedResults = new List<IResult>();
427      CollectResultsRecursively("", aggregatedResults, flattenedResults);
428      return flattenedResults;
429    }
430
431    private IEnumerable<IResult> ExtractAndAggregateClassificationSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
432      Dictionary<string, List<IClassificationSolution>> resultSolutions = new Dictionary<string, List<IClassificationSolution>>();
433      foreach (var result in resultCollections) {
434        var classificationSolution = result.Value as IClassificationSolution;
435        if (classificationSolution != null) {
436          if (resultSolutions.ContainsKey(result.Key)) {
437            resultSolutions[result.Key].Add(classificationSolution);
438          } else {
439            resultSolutions.Add(result.Key, new List<IClassificationSolution>() { classificationSolution });
440          }
441        }
442      }
443      var aggregatedResults = new List<IResult>();
444      foreach (KeyValuePair<string, List<IClassificationSolution>> solutions in resultSolutions) {
445        // clone manually to correctly clone references between cloned root objects
446        Cloner cloner = new Cloner();
447        var problemDataClone = (IClassificationProblemData)cloner.Clone(Problem.ProblemData);
448        // set partitions of problem data clone correctly
449        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
450        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
451        // clone models
452        var ensembleSolution = new ClassificationEnsembleSolution(
453          solutions.Value.Select(x => cloner.Clone(x.Model)),
454          problemDataClone,
455          solutions.Value.Select(x => cloner.Clone(x.ProblemData.TrainingPartition)),
456          solutions.Value.Select(x => cloner.Clone(x.ProblemData.TestPartition)));
457
458        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
459      }
460      List<IResult> flattenedResults = new List<IResult>();
461      CollectResultsRecursively("", aggregatedResults, flattenedResults);
462      return flattenedResults;
463    }
464
465    private void CollectResultsRecursively(string path, IEnumerable<IResult> results, IList<IResult> flattenedResults) {
466      foreach (IResult result in results) {
467        flattenedResults.Add(new Result(path + result.Name, result.Value));
468        ResultCollection childCollection = result.Value as ResultCollection;
469        if (childCollection != null) {
470          CollectResultsRecursively(path + result.Name + ".", childCollection, flattenedResults);
471        }
472      }
473    }
474
475    private static IEnumerable<IResult> ExtractAndAggregateResults<T>(IEnumerable<KeyValuePair<string, IItem>> results)
476  where T : class, IItem, new() {
477      Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
478      foreach (var resultValue in results.Where(r => r.Value.GetType() == typeof(T))) {
479        if (!resultValues.ContainsKey(resultValue.Key))
480          resultValues[resultValue.Key] = new List<double>();
481        resultValues[resultValue.Key].Add(ConvertToDouble(resultValue.Value));
482      }
483
484      DoubleValue doubleValue;
485      if (typeof(T) == typeof(PercentValue))
486        doubleValue = new PercentValue();
487      else if (typeof(T) == typeof(DoubleValue))
488        doubleValue = new DoubleValue();
489      else if (typeof(T) == typeof(IntValue))
490        doubleValue = new DoubleValue();
491      else
492        throw new NotSupportedException();
493
494      List<IResult> aggregatedResults = new List<IResult>();
495      foreach (KeyValuePair<string, List<double>> resultValue in resultValues) {
496        doubleValue.Value = resultValue.Value.Average();
497        aggregatedResults.Add(new Result(resultValue.Key + " (average)", (IItem)doubleValue.Clone()));
498        doubleValue.Value = resultValue.Value.StandardDeviation();
499        aggregatedResults.Add(new Result(resultValue.Key + " (std.dev.)", (IItem)doubleValue.Clone()));
500      }
501      return aggregatedResults;
502    }
503
504    private static double ConvertToDouble(IItem item) {
505      if (item is DoubleValue) return ((DoubleValue)item).Value;
506      else if (item is IntValue) return ((IntValue)item).Value;
507      else throw new NotSupportedException("Could not convert any item type to double");
508    }
509    #endregion
510
511    #region events
512    private void RegisterEvents() {
513      Folds.ValueChanged += new EventHandler(Folds_ValueChanged);
514      SamplesStart.ValueChanged += new EventHandler(SamplesStart_ValueChanged);
515      SamplesEnd.ValueChanged += new EventHandler(SamplesEnd_ValueChanged);
516      RegisterClonedAlgorithmsEvents();
517    }
518    private void Folds_ValueChanged(object sender, EventArgs e) {
519      if (ExecutionState != ExecutionState.Prepared)
520        throw new InvalidOperationException("Can not change number of folds if the execution state is not prepared.");
521    }
522
523    private bool samplesChanged = false;
524    private void SamplesStart_ValueChanged(object sender, EventArgs e) {
525      samplesChanged = true;
526      if (Problem != null) Problem.ProblemData.TrainingPartition.Start = SamplesStart.Value;
527      samplesChanged = false;
528    }
529    private void SamplesEnd_ValueChanged(object sender, EventArgs e) {
530      samplesChanged = true;
531      if (Problem != null) Problem.ProblemData.TrainingPartition.End = SamplesEnd.Value;
532      samplesChanged = false;
533    }
534
535    #region template algorithms events
536    public event EventHandler AlgorithmChanged;
537    private void OnAlgorithmChanged() {
538      EventHandler handler = AlgorithmChanged;
539      if (handler != null) handler(this, EventArgs.Empty);
540      OnProblemChanged();
541      if (Problem == null) ExecutionState = ExecutionState.Stopped;
542    }
543    private void RegisterAlgorithmEvents() {
544      algorithm.ProblemChanged += new EventHandler(Algorithm_ProblemChanged);
545      algorithm.ExecutionStateChanged += new EventHandler(Algorithm_ExecutionStateChanged);
546    }
547    private void DeregisterAlgorithmEvents() {
548      algorithm.ProblemChanged -= new EventHandler(Algorithm_ProblemChanged);
549      algorithm.ExecutionStateChanged -= new EventHandler(Algorithm_ExecutionStateChanged);
550    }
551    private void Algorithm_ProblemChanged(object sender, EventArgs e) {
552      if (algorithm.Problem != null && !(algorithm.Problem is IDataAnalysisProblem)) {
553        algorithm.Problem = problem;
554        throw new ArgumentException("A cross validation algorithm can only contain DataAnalysisProblems.");
555      }
556      algorithm.Problem.Reset += (x, y) => OnProblemChanged();
557      problem = (IDataAnalysisProblem)algorithm.Problem;
558      OnProblemChanged();
559    }
560    public event EventHandler ProblemChanged;
561    private void OnProblemChanged() {
562      EventHandler handler = ProblemChanged;
563      if (handler != null) handler(this, EventArgs.Empty);
564      if (samplesChanged) return;
565
566      SamplesStart.Value = 0;
567      if (Problem != null) {
568        SamplesEnd.Value = Problem.ProblemData.Dataset.Rows;
569
570        DataAnalysisProblemData problemData = Problem.ProblemData as DataAnalysisProblemData;
571        if (problemData != null) {
572          problemData.TrainingPartitionParameter.Hidden = true;
573          problemData.TestPartitionParameter.Hidden = true;
574        }
575        ISymbolicDataAnalysisProblem symbolicProblem = Problem as ISymbolicDataAnalysisProblem;
576        if (symbolicProblem != null) {
577          symbolicProblem.FitnessCalculationPartitionParameter.Hidden = true;
578          symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
579          symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
580          symbolicProblem.ValidationPartitionParameter.Hidden = true;
581          symbolicProblem.ValidationPartition.Start = 0;
582          symbolicProblem.ValidationPartition.End = 0;
583        }
584      } else
585        SamplesEnd.Value = 0;
586
587      SamplesStart_ValueChanged(this, EventArgs.Empty);
588      SamplesEnd_ValueChanged(this, EventArgs.Empty);
589    }
590
591    private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
592      switch (Algorithm.ExecutionState) {
593        case ExecutionState.Prepared: OnPrepared();
594          break;
595        case ExecutionState.Started: throw new InvalidOperationException("Algorithm template can not be started.");
596        case ExecutionState.Paused: throw new InvalidOperationException("Algorithm template can not be paused.");
597        case ExecutionState.Stopped: OnStopped();
598          break;
599      }
600    }
601    #endregion
602
603    #region clonedAlgorithms events
604    private void RegisterClonedAlgorithmsEvents() {
605      clonedAlgorithms.ItemsAdded += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
606      clonedAlgorithms.ItemsRemoved += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
607      clonedAlgorithms.CollectionReset += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
608      foreach (IAlgorithm algorithm in clonedAlgorithms)
609        RegisterClonedAlgorithmEvents(algorithm);
610    }
611    private void DeregisterClonedAlgorithmsEvents() {
612      clonedAlgorithms.ItemsAdded -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
613      clonedAlgorithms.ItemsRemoved -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
614      clonedAlgorithms.CollectionReset -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
615      foreach (IAlgorithm algorithm in clonedAlgorithms)
616        DeregisterClonedAlgorithmEvents(algorithm);
617    }
618    private void ClonedAlgorithms_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
619      foreach (IAlgorithm algorithm in e.Items)
620        RegisterClonedAlgorithmEvents(algorithm);
621    }
622    private void ClonedAlgorithms_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
623      foreach (IAlgorithm algorithm in e.Items)
624        DeregisterClonedAlgorithmEvents(algorithm);
625    }
626    private void ClonedAlgorithms_CollectionReset(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
627      foreach (IAlgorithm algorithm in e.OldItems)
628        DeregisterClonedAlgorithmEvents(algorithm);
629      foreach (IAlgorithm algorithm in e.Items)
630        RegisterClonedAlgorithmEvents(algorithm);
631    }
632    private void RegisterClonedAlgorithmEvents(IAlgorithm algorithm) {
633      algorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
634      algorithm.ExecutionTimeChanged += new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
635      algorithm.Started += new EventHandler(ClonedAlgorithm_Started);
636      algorithm.Paused += new EventHandler(ClonedAlgorithm_Paused);
637      algorithm.Stopped += new EventHandler(ClonedAlgorithm_Stopped);
638    }
639    private void DeregisterClonedAlgorithmEvents(IAlgorithm algorithm) {
640      algorithm.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
641      algorithm.ExecutionTimeChanged -= new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
642      algorithm.Started -= new EventHandler(ClonedAlgorithm_Started);
643      algorithm.Paused -= new EventHandler(ClonedAlgorithm_Paused);
644      algorithm.Stopped -= new EventHandler(ClonedAlgorithm_Stopped);
645    }
646    private void ClonedAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
647      OnExceptionOccurred(e.Value);
648    }
649    private void ClonedAlgorithm_ExecutionTimeChanged(object sender, EventArgs e) {
650      OnExecutionTimeChanged();
651    }
652
653    private readonly object locker = new object();
654    private void ClonedAlgorithm_Started(object sender, EventArgs e) {
655      lock (locker) {
656        IAlgorithm algorithm = sender as IAlgorithm;
657        if (algorithm != null && !results.ContainsKey(algorithm.Name))
658          results.Add(new Result(algorithm.Name, "Contains results for the specific fold.", algorithm.Results));
659
660        if (startPending) {
661          int startedAlgorithms = clonedAlgorithms.Count(alg => alg.ExecutionState == ExecutionState.Started);
662          if (startedAlgorithms == NumberOfWorkers.Value ||
663             clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Prepared))
664            startPending = false;
665
666          if (pausePending) PauseAllClonedAlgorithms();
667          if (stopPending) StopAllClonedAlgorithms();
668        }
669      }
670    }
671
672    private void ClonedAlgorithm_Paused(object sender, EventArgs e) {
673      lock (locker) {
674        if (pausePending && clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started))
675          OnPaused();
676      }
677    }
678
679    private void ClonedAlgorithm_Stopped(object sender, EventArgs e) {
680      lock (locker) {
681        if (!stopPending && ExecutionState == ExecutionState.Started) {
682          IAlgorithm preparedAlgorithm = clonedAlgorithms.Where(alg => alg.ExecutionState == ExecutionState.Prepared ||
683                                                                       alg.ExecutionState == ExecutionState.Paused).FirstOrDefault();
684          if (preparedAlgorithm != null) preparedAlgorithm.Start();
685        }
686        if (ExecutionState != ExecutionState.Stopped) {
687          if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped))
688            OnStopped();
689          else if (stopPending &&
690                   clonedAlgorithms.All(
691                     alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped))
692            OnStopped();
693        }
694      }
695    }
696    #endregion
697    #endregion
698
699    #region event firing
700    public event EventHandler ExecutionStateChanged;
701    private void OnExecutionStateChanged() {
702      EventHandler handler = ExecutionStateChanged;
703      if (handler != null) handler(this, EventArgs.Empty);
704    }
705    public event EventHandler ExecutionTimeChanged;
706    private void OnExecutionTimeChanged() {
707      EventHandler handler = ExecutionTimeChanged;
708      if (handler != null) handler(this, EventArgs.Empty);
709    }
710    public event EventHandler Prepared;
711    private void OnPrepared() {
712      ExecutionState = ExecutionState.Prepared;
713      EventHandler handler = Prepared;
714      if (handler != null) handler(this, EventArgs.Empty);
715      OnExecutionTimeChanged();
716    }
717    public event EventHandler Started;
718    private void OnStarted() {
719      startPending = false;
720      ExecutionState = ExecutionState.Started;
721      EventHandler handler = Started;
722      if (handler != null) handler(this, EventArgs.Empty);
723    }
724    public event EventHandler Paused;
725    private void OnPaused() {
726      pausePending = false;
727      ExecutionState = ExecutionState.Paused;
728      EventHandler handler = Paused;
729      if (handler != null) handler(this, EventArgs.Empty);
730    }
731    public event EventHandler Stopped;
732    private void OnStopped() {
733      stopPending = false;
734      Dictionary<string, IItem> collectedResults = new Dictionary<string, IItem>();
735      AggregateResultValues(collectedResults);
736      results.AddRange(collectedResults.Select(x => new Result(x.Key, x.Value)).Cast<IResult>().ToArray());
737      runsCounter++;
738      runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
739      ExecutionState = ExecutionState.Stopped;
740      EventHandler handler = Stopped;
741      if (handler != null) handler(this, EventArgs.Empty);
742    }
743    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
744    private void OnExceptionOccurred(Exception exception) {
745      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
746      if (handler != null) handler(this, new EventArgs<Exception>(exception));
747    }
748    public event EventHandler StoreAlgorithmInEachRunChanged;
749    private void OnStoreAlgorithmInEachRunChanged() {
750      EventHandler handler = StoreAlgorithmInEachRunChanged;
751      if (handler != null) handler(this, EventArgs.Empty);
752    }
753    #endregion
754  }
755}
Note: See TracBrowser for help on using the repository browser.