source: branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs @ 14870

Last change on this file since 14870 was 14870, checked in by gkronber, 2 years ago

#2699: merged changesets from trunk to branch

File size: 32.9 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Drawing;
25using System.Linq;
26using System.Threading;
27using HeuristicLab.Collections;
28using HeuristicLab.Common;
29using HeuristicLab.Core;
30using HeuristicLab.Data;
31using HeuristicLab.Optimization;
32using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
33using HeuristicLab.Problems.DataAnalysis;
34using HeuristicLab.Problems.DataAnalysis.Symbolic;
35using HeuristicLab.Random;
36
37namespace HeuristicLab.Algorithms.DataAnalysis {
38  [Item("Cross Validation (CV)", "Cross-validation wrapper for data analysis algorithms.")]
39  [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 100)]
40  [StorableClass]
41  public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent {
42    public CrossValidation()
43      : base() {
44      name = ItemName;
45      description = ItemDescription;
46
47      executionState = ExecutionState.Stopped;
48      runs = new RunCollection { OptimizerName = name };
49      runsCounter = 0;
50
51      algorithm = null;
52      clonedAlgorithms = new ItemCollection<IAlgorithm>();
53      results = new ResultCollection();
54
55      folds = new IntValue(2);
56      numberOfWorkers = new IntValue(1);
57      samplesStart = new IntValue(0);
58      samplesEnd = new IntValue(0);
59      shuffleSamples = new BoolValue(false);
60      storeAlgorithmInEachRun = false;
61
62      RegisterEvents();
63      if (Algorithm != null) RegisterAlgorithmEvents();
64    }
65
66    public string Filename { get; set; }
67
68    #region persistence and cloning
69    [StorableConstructor]
70    private CrossValidation(bool deserializing)
71      : base(deserializing) {
72    }
73    [StorableHook(HookType.AfterDeserialization)]
74    private void AfterDeserialization() {
75      RegisterEvents();
76      if (Algorithm != null) RegisterAlgorithmEvents();
77    }
78
79    private CrossValidation(CrossValidation original, Cloner cloner)
80      : base(original, cloner) {
81      executionState = original.executionState;
82      storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
83      runs = cloner.Clone(original.runs);
84      runsCounter = original.runsCounter;
85      algorithm = cloner.Clone(original.algorithm);
86      clonedAlgorithms = cloner.Clone(original.clonedAlgorithms);
87      results = cloner.Clone(original.results);
88
89      folds = cloner.Clone(original.folds);
90      numberOfWorkers = cloner.Clone(original.numberOfWorkers);
91      samplesStart = cloner.Clone(original.samplesStart);
92      samplesEnd = cloner.Clone(original.samplesEnd);
93      shuffleSamples = cloner.Clone(original.shuffleSamples);
94      RegisterEvents();
95      if (Algorithm != null) RegisterAlgorithmEvents();
96    }
97    public override IDeepCloneable Clone(Cloner cloner) {
98      return new CrossValidation(this, cloner);
99    }
100
101    #endregion
102
103    #region properties
104    [Storable]
105    private IAlgorithm algorithm;
106    public IAlgorithm Algorithm {
107      get { return algorithm; }
108      set {
109        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
110          throw new InvalidOperationException("Changing the algorithm is only allowed if the CrossValidation is stopped or prepared.");
111        if (algorithm != value) {
112          if (value != null && value.Problem != null && !(value.Problem is IDataAnalysisProblem))
113            throw new ArgumentException("Only algorithms with a DataAnalysisProblem could be used for the cross validation.");
114          if (algorithm != null) DeregisterAlgorithmEvents();
115          algorithm = value;
116          Parameters.Clear();
117
118          if (algorithm != null) {
119            algorithm.StoreAlgorithmInEachRun = false;
120            RegisterAlgorithmEvents();
121            algorithm.Prepare(true);
122            Parameters.AddRange(algorithm.Parameters);
123          }
124          OnAlgorithmChanged();
125          Prepare();
126        }
127      }
128    }
129
130
131    [Storable]
132    private IDataAnalysisProblem problem;
133    public IDataAnalysisProblem Problem {
134      get {
135        if (algorithm == null)
136          return null;
137        return (IDataAnalysisProblem)algorithm.Problem;
138      }
139      set {
140        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
141          throw new InvalidOperationException("Changing the problem is only allowed if the CrossValidation is stopped or prepared.");
142        if (algorithm == null) throw new ArgumentNullException("Could not set a problem before an algorithm was set.");
143        algorithm.Problem = value;
144        problem = value;
145      }
146    }
147
148    IProblem IAlgorithm.Problem {
149      get { return Problem; }
150      set {
151        if (value != null && !ProblemType.IsInstanceOfType(value))
152          throw new ArgumentException("Only DataAnalysisProblems could be used for the cross validation.");
153        Problem = (IDataAnalysisProblem)value;
154      }
155    }
156    public Type ProblemType {
157      get { return typeof(IDataAnalysisProblem); }
158    }
159
160    [Storable]
161    private ItemCollection<IAlgorithm> clonedAlgorithms;
162
163    public IEnumerable<IOptimizer> NestedOptimizers {
164      get {
165        if (Algorithm == null) yield break;
166        yield return Algorithm;
167      }
168    }
169
170    [Storable]
171    private ResultCollection results;
172    public ResultCollection Results {
173      get { return results; }
174    }
175    [Storable]
176    private BoolValue shuffleSamples;
177    public BoolValue ShuffleSamples {
178      get { return shuffleSamples; }
179    }
180    [Storable]
181    private IntValue folds;
182    public IntValue Folds {
183      get { return folds; }
184    }
185    [Storable]
186    private IntValue samplesStart;
187    public IntValue SamplesStart {
188      get { return samplesStart; }
189    }
190    [Storable]
191    private IntValue samplesEnd;
192    public IntValue SamplesEnd {
193      get { return samplesEnd; }
194    }
195    [Storable]
196    private IntValue numberOfWorkers;
197    public IntValue NumberOfWorkers {
198      get { return numberOfWorkers; }
199    }
200
201    [Storable]
202    private bool storeAlgorithmInEachRun;
203    public bool StoreAlgorithmInEachRun {
204      get { return storeAlgorithmInEachRun; }
205      set {
206        if (storeAlgorithmInEachRun != value) {
207          storeAlgorithmInEachRun = value;
208          OnStoreAlgorithmInEachRunChanged();
209        }
210      }
211    }
212
213    [Storable]
214    private int runsCounter;
215    [Storable]
216    private RunCollection runs;
217    public RunCollection Runs {
218      get { return runs; }
219    }
220
221    [Storable]
222    private ExecutionState executionState;
223    public ExecutionState ExecutionState {
224      get { return executionState; }
225      private set {
226        if (executionState != value) {
227          executionState = value;
228          OnExecutionStateChanged();
229          OnItemImageChanged();
230        }
231      }
232    }
233    public static new Image StaticItemImage {
234      get { return HeuristicLab.Common.Resources.VSImageLibrary.Event; }
235    }
236    public override Image ItemImage {
237      get {
238        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
239        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
240        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
241        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
242        else return base.ItemImage;
243      }
244    }
245
246    public TimeSpan ExecutionTime {
247      get {
248        if (ExecutionState != ExecutionState.Prepared)
249          return TimeSpan.FromMilliseconds(clonedAlgorithms.Select(x => x.ExecutionTime.TotalMilliseconds).Sum());
250        return TimeSpan.Zero;
251      }
252    }
253    #endregion
254
255    protected override void OnNameChanged() {
256      base.OnNameChanged();
257      Runs.OptimizerName = Name;
258    }
259
260    public void Prepare() {
261      if (ExecutionState == ExecutionState.Started)
262        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
263      results.Clear();
264      clonedAlgorithms.Clear();
265      if (Algorithm != null) {
266        Algorithm.Prepare();
267        if (Algorithm.ExecutionState == ExecutionState.Prepared) OnPrepared();
268      }
269    }
270    public void Prepare(bool clearRuns) {
271      if (clearRuns) runs.Clear();
272      Prepare();
273    }
274
275    public void Start() {
276      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
277        throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));
278
279      if (Algorithm != null) {
280        //create cloned algorithms
281        if (clonedAlgorithms.Count == 0) {
282          int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / Folds.Value;
283          IDataset shuffledDataset = null;
284          for (int i = 0; i < Folds.Value; i++) {
285            var cloner = new Cloner();
286            if (ShuffleSamples.Value) {
287              var dataAnalysisProblem = (IDataAnalysisProblem)algorithm.Problem;
288              var dataset = (Dataset)dataAnalysisProblem.ProblemData.Dataset;
289              shuffledDataset = shuffledDataset ?? dataset.Shuffle(new FastRandom());
290              cloner.RegisterClonedObject(dataset, shuffledDataset);
291            }
292            IAlgorithm clonedAlgorithm = cloner.Clone(Algorithm);
293            clonedAlgorithm.Name = algorithm.Name + " Fold " + i;
294            IDataAnalysisProblem problem = clonedAlgorithm.Problem as IDataAnalysisProblem;
295            ISymbolicDataAnalysisProblem symbolicProblem = problem as ISymbolicDataAnalysisProblem;
296
297            int testStart = (i * testSamplesCount) + SamplesStart.Value;
298            int testEnd = (i + 1) == Folds.Value ? SamplesEnd.Value : (i + 1) * testSamplesCount + SamplesStart.Value;
299
300            problem.ProblemData.TrainingPartition.Start = SamplesStart.Value;
301            problem.ProblemData.TrainingPartition.End = SamplesEnd.Value;
302            problem.ProblemData.TestPartition.Start = testStart;
303            problem.ProblemData.TestPartition.End = testEnd;
304            DataAnalysisProblemData problemData = problem.ProblemData as DataAnalysisProblemData;
305            if (problemData != null) {
306              problemData.TrainingPartitionParameter.Hidden = false;
307              problemData.TestPartitionParameter.Hidden = false;
308            }
309
310            if (symbolicProblem != null) {
311              symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
312              symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
313            }
314            clonedAlgorithm.Prepare();
315            clonedAlgorithms.Add(clonedAlgorithm);
316          }
317        }
318
319        //start prepared or paused cloned algorithms
320        int startedAlgorithms = 0;
321        foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
322          if (startedAlgorithms < NumberOfWorkers.Value) {
323            if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared ||
324                clonedAlgorithm.ExecutionState == ExecutionState.Paused) {
325
326              // start and wait until the alg is started
327              using (var signal = new ManualResetEvent(false)) {
328                EventHandler signalSetter = (sender, args) => { signal.Set(); };
329                clonedAlgorithm.Started += signalSetter;
330                clonedAlgorithm.Start();
331                signal.WaitOne();
332                clonedAlgorithm.Started -= signalSetter;
333
334                startedAlgorithms++;
335              }
336            }
337          }
338        }
339        OnStarted();
340      }
341    }
342
343    private bool pausePending;
344    public void Pause() {
345      if (ExecutionState != ExecutionState.Started)
346        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
347      if (!pausePending) {
348        pausePending = true;
349        PauseAllClonedAlgorithms();
350      }
351    }
352    private void PauseAllClonedAlgorithms() {
353      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
354        if (clonedAlgorithm.ExecutionState == ExecutionState.Started)
355          clonedAlgorithm.Pause();
356      }
357    }
358
359    private bool stopPending;
360    public void Stop() {
361      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
362        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".",
363                                                          ExecutionState));
364      if (!stopPending) {
365        stopPending = true;
366        StopAllClonedAlgorithms();
367      }
368    }
369    private void StopAllClonedAlgorithms() {
370      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
371        if (clonedAlgorithm.ExecutionState == ExecutionState.Started ||
372            clonedAlgorithm.ExecutionState == ExecutionState.Paused)
373          clonedAlgorithm.Stop();
374      }
375    }
376
377    #region collect parameters and results
378    public override void CollectParameterValues(IDictionary<string, IItem> values) {
379      values.Add("Algorithm Name", new StringValue(Name));
380      values.Add("Algorithm Type", new StringValue(GetType().GetPrettyName()));
381      values.Add("Folds", new IntValue(Folds.Value));
382
383      if (algorithm != null) {
384        values.Add("CrossValidation Algorithm Name", new StringValue(Algorithm.Name));
385        values.Add("CrossValidation Algorithm Type", new StringValue(Algorithm.GetType().GetPrettyName()));
386        base.CollectParameterValues(values);
387      }
388      if (Problem != null) {
389        values.Add("Problem Name", new StringValue(Problem.Name));
390        values.Add("Problem Type", new StringValue(Problem.GetType().GetPrettyName()));
391        Problem.CollectParameterValues(values);
392      }
393    }
394
395    public void CollectResultValues(IDictionary<string, IItem> results) {
396      var clonedResults = (ResultCollection)this.results.Clone();
397      foreach (var result in clonedResults) {
398        results.Add(result.Name, result.Value);
399      }
400    }
401
402    private void AggregateResultValues(IDictionary<string, IItem> results) {
403      IEnumerable<IRun> runs = clonedAlgorithms.Select(alg => alg.Runs.FirstOrDefault()).Where(run => run != null);
404      IEnumerable<KeyValuePair<string, IItem>> resultCollections = runs.Where(x => x != null).SelectMany(x => x.Results).ToList();
405
406      foreach (IResult result in ExtractAndAggregateResults<IntValue>(resultCollections))
407        results.Add(result.Name, result.Value);
408      foreach (IResult result in ExtractAndAggregateResults<DoubleValue>(resultCollections))
409        results.Add(result.Name, result.Value);
410      foreach (IResult result in ExtractAndAggregateResults<PercentValue>(resultCollections))
411        results.Add(result.Name, result.Value);
412      foreach (IResult result in ExtractAndAggregateRegressionSolutions(resultCollections)) {
413        results.Add(result.Name, result.Value);
414      }
415      foreach (IResult result in ExtractAndAggregateClassificationSolutions(resultCollections)) {
416        results.Add(result.Name, result.Value);
417      }
418      results.Add("Execution Time", new TimeSpanValue(this.ExecutionTime));
419      results.Add("CrossValidation Folds", new RunCollection(runs));
420    }
421
422    private IEnumerable<IResult> ExtractAndAggregateRegressionSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
423      Dictionary<string, List<IRegressionSolution>> resultSolutions = new Dictionary<string, List<IRegressionSolution>>();
424      foreach (var result in resultCollections) {
425        var regressionSolution = result.Value as IRegressionSolution;
426        if (regressionSolution != null) {
427          if (resultSolutions.ContainsKey(result.Key)) {
428            resultSolutions[result.Key].Add(regressionSolution);
429          } else {
430            resultSolutions.Add(result.Key, new List<IRegressionSolution>() { regressionSolution });
431          }
432        }
433      }
434      List<IResult> aggregatedResults = new List<IResult>();
435      foreach (KeyValuePair<string, List<IRegressionSolution>> solutions in resultSolutions) {
436        // clone manually to correctly clone references between cloned root objects
437        Cloner cloner = new Cloner();
438        var problemDataClone = (IRegressionProblemData)cloner.Clone(Problem.ProblemData);
439        // set partitions of problem data clone correctly
440        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
441        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
442        // clone models
443        var ensembleSolution = new RegressionEnsembleSolution(problemDataClone);
444        ensembleSolution.AddRegressionSolutions(solutions.Value);
445
446        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
447      }
448      List<IResult> flattenedResults = new List<IResult>();
449      CollectResultsRecursively("", aggregatedResults, flattenedResults);
450      return flattenedResults;
451    }
452
453    private IEnumerable<IResult> ExtractAndAggregateClassificationSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
454      Dictionary<string, List<IClassificationSolution>> resultSolutions = new Dictionary<string, List<IClassificationSolution>>();
455      foreach (var result in resultCollections) {
456        var classificationSolution = result.Value as IClassificationSolution;
457        if (classificationSolution != null) {
458          if (resultSolutions.ContainsKey(result.Key)) {
459            resultSolutions[result.Key].Add(classificationSolution);
460          } else {
461            resultSolutions.Add(result.Key, new List<IClassificationSolution>() { classificationSolution });
462          }
463        }
464      }
465      var aggregatedResults = new List<IResult>();
466      foreach (KeyValuePair<string, List<IClassificationSolution>> solutions in resultSolutions) {
467        // at least one algorithm (GBT with logistic regression loss) produces a classification solution even though the original problem is a regression problem.
468        var targetVariable = solutions.Value.First().ProblemData.TargetVariable;
469        var problemDataClone = new ClassificationProblemData(Problem.ProblemData.Dataset,
470          Problem.ProblemData.AllowedInputVariables, targetVariable);
471        // set partitions of problem data clone correctly
472        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
473        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
474        // clone models
475        var ensembleSolution = new ClassificationEnsembleSolution(problemDataClone);
476        ensembleSolution.AddClassificationSolutions(solutions.Value);
477
478        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
479      }
480      List<IResult> flattenedResults = new List<IResult>();
481      CollectResultsRecursively("", aggregatedResults, flattenedResults);
482      return flattenedResults;
483    }
484
485    private void CollectResultsRecursively(string path, IEnumerable<IResult> results, IList<IResult> flattenedResults) {
486      foreach (IResult result in results) {
487        flattenedResults.Add(new Result(path + result.Name, result.Value));
488        ResultCollection childCollection = result.Value as ResultCollection;
489        if (childCollection != null) {
490          CollectResultsRecursively(path + result.Name + ".", childCollection, flattenedResults);
491        }
492      }
493    }
494
495    private static IEnumerable<IResult> ExtractAndAggregateResults<T>(IEnumerable<KeyValuePair<string, IItem>> results)
496  where T : class, IItem, new() {
497      Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
498      foreach (var resultValue in results.Where(r => r.Value.GetType() == typeof(T))) {
499        if (!resultValues.ContainsKey(resultValue.Key))
500          resultValues[resultValue.Key] = new List<double>();
501        resultValues[resultValue.Key].Add(ConvertToDouble(resultValue.Value));
502      }
503
504      DoubleValue doubleValue;
505      if (typeof(T) == typeof(PercentValue))
506        doubleValue = new PercentValue();
507      else if (typeof(T) == typeof(DoubleValue))
508        doubleValue = new DoubleValue();
509      else if (typeof(T) == typeof(IntValue))
510        doubleValue = new DoubleValue();
511      else
512        throw new NotSupportedException();
513
514      List<IResult> aggregatedResults = new List<IResult>();
515      foreach (KeyValuePair<string, List<double>> resultValue in resultValues) {
516        doubleValue.Value = resultValue.Value.Average();
517        aggregatedResults.Add(new Result(resultValue.Key + " (average)", (IItem)doubleValue.Clone()));
518        doubleValue.Value = resultValue.Value.StandardDeviation();
519        aggregatedResults.Add(new Result(resultValue.Key + " (std.dev.)", (IItem)doubleValue.Clone()));
520      }
521      return aggregatedResults;
522    }
523
524    private static double ConvertToDouble(IItem item) {
525      if (item is DoubleValue) return ((DoubleValue)item).Value;
526      else if (item is IntValue) return ((IntValue)item).Value;
527      else throw new NotSupportedException("Could not convert any item type to double");
528    }
529    #endregion
530
531    #region events
532    private void RegisterEvents() {
533      Folds.ValueChanged += new EventHandler(Folds_ValueChanged);
534      RegisterClonedAlgorithmsEvents();
535    }
536    private void Folds_ValueChanged(object sender, EventArgs e) {
537      if (ExecutionState != ExecutionState.Prepared)
538        throw new InvalidOperationException("Can not change number of folds if the execution state is not prepared.");
539    }
540
541
542    #region template algorithms events
543    public event EventHandler AlgorithmChanged;
544    private void OnAlgorithmChanged() {
545      EventHandler handler = AlgorithmChanged;
546      if (handler != null) handler(this, EventArgs.Empty);
547      OnProblemChanged();
548      if (Problem == null) ExecutionState = ExecutionState.Stopped;
549    }
550    private void RegisterAlgorithmEvents() {
551      algorithm.ProblemChanged += new EventHandler(Algorithm_ProblemChanged);
552      algorithm.ExecutionStateChanged += new EventHandler(Algorithm_ExecutionStateChanged);
553      if (Problem != null) Problem.Reset += new EventHandler(Problem_Reset);
554    }
555    private void DeregisterAlgorithmEvents() {
556      algorithm.ProblemChanged -= new EventHandler(Algorithm_ProblemChanged);
557      algorithm.ExecutionStateChanged -= new EventHandler(Algorithm_ExecutionStateChanged);
558      if (Problem != null) Problem.Reset -= new EventHandler(Problem_Reset);
559    }
560    private void Algorithm_ProblemChanged(object sender, EventArgs e) {
561      if (algorithm.Problem != null && !(algorithm.Problem is IDataAnalysisProblem)) {
562        algorithm.Problem = problem;
563        throw new ArgumentException("A cross validation algorithm can only contain DataAnalysisProblems.");
564      }
565      if (problem != null) problem.Reset -= new EventHandler(Problem_Reset);
566      problem = (IDataAnalysisProblem)algorithm.Problem;
567      if (problem != null) problem.Reset += new EventHandler(Problem_Reset);
568      OnProblemChanged();
569    }
570    public event EventHandler ProblemChanged;
571    private void OnProblemChanged() {
572      EventHandler handler = ProblemChanged;
573      if (handler != null) handler(this, EventArgs.Empty);
574      ConfigureProblem();
575    }
576
577    private void Problem_Reset(object sender, EventArgs e) {
578      ConfigureProblem();
579    }
580
581    private void ConfigureProblem() {
582      SamplesStart.Value = 0;
583      if (Problem != null) {
584        SamplesEnd.Value = Problem.ProblemData.Dataset.Rows;
585
586        DataAnalysisProblemData problemData = Problem.ProblemData as DataAnalysisProblemData;
587        if (problemData != null) {
588          problemData.TrainingPartitionParameter.Hidden = true;
589          problemData.TestPartitionParameter.Hidden = true;
590        }
591        ISymbolicDataAnalysisProblem symbolicProblem = Problem as ISymbolicDataAnalysisProblem;
592        if (symbolicProblem != null) {
593          symbolicProblem.FitnessCalculationPartitionParameter.Hidden = true;
594          symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
595          symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
596          symbolicProblem.ValidationPartitionParameter.Hidden = true;
597          symbolicProblem.ValidationPartition.Start = 0;
598          symbolicProblem.ValidationPartition.End = 0;
599        }
600      } else
601        SamplesEnd.Value = 0;
602    }
603
604    private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
605      switch (Algorithm.ExecutionState) {
606        case ExecutionState.Prepared:
607          OnPrepared();
608          break;
609        case ExecutionState.Started: throw new InvalidOperationException("Algorithm template can not be started.");
610        case ExecutionState.Paused: throw new InvalidOperationException("Algorithm template can not be paused.");
611        case ExecutionState.Stopped:
612          OnStopped();
613          break;
614      }
615    }
616    #endregion
617
618    #region clonedAlgorithms events
619    private void RegisterClonedAlgorithmsEvents() {
620      clonedAlgorithms.ItemsAdded += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
621      clonedAlgorithms.ItemsRemoved += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
622      clonedAlgorithms.CollectionReset += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
623      foreach (IAlgorithm algorithm in clonedAlgorithms)
624        RegisterClonedAlgorithmEvents(algorithm);
625    }
626    private void DeregisterClonedAlgorithmsEvents() {
627      clonedAlgorithms.ItemsAdded -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
628      clonedAlgorithms.ItemsRemoved -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
629      clonedAlgorithms.CollectionReset -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
630      foreach (IAlgorithm algorithm in clonedAlgorithms)
631        DeregisterClonedAlgorithmEvents(algorithm);
632    }
633    private void ClonedAlgorithms_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
634      foreach (IAlgorithm algorithm in e.Items)
635        RegisterClonedAlgorithmEvents(algorithm);
636    }
637    private void ClonedAlgorithms_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
638      foreach (IAlgorithm algorithm in e.Items)
639        DeregisterClonedAlgorithmEvents(algorithm);
640    }
641    private void ClonedAlgorithms_CollectionReset(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
642      foreach (IAlgorithm algorithm in e.OldItems)
643        DeregisterClonedAlgorithmEvents(algorithm);
644      foreach (IAlgorithm algorithm in e.Items)
645        RegisterClonedAlgorithmEvents(algorithm);
646    }
647    private void RegisterClonedAlgorithmEvents(IAlgorithm algorithm) {
648      algorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
649      algorithm.ExecutionTimeChanged += new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
650      algorithm.Started += new EventHandler(ClonedAlgorithm_Started);
651      algorithm.Paused += new EventHandler(ClonedAlgorithm_Paused);
652      algorithm.Stopped += new EventHandler(ClonedAlgorithm_Stopped);
653    }
654    private void DeregisterClonedAlgorithmEvents(IAlgorithm algorithm) {
655      algorithm.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
656      algorithm.ExecutionTimeChanged -= new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
657      algorithm.Started -= new EventHandler(ClonedAlgorithm_Started);
658      algorithm.Paused -= new EventHandler(ClonedAlgorithm_Paused);
659      algorithm.Stopped -= new EventHandler(ClonedAlgorithm_Stopped);
660    }
661    private void ClonedAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
662      OnExceptionOccurred(e.Value);
663    }
664    private void ClonedAlgorithm_ExecutionTimeChanged(object sender, EventArgs e) {
665      OnExecutionTimeChanged();
666    }
667
668    private readonly object locker = new object();
669    private readonly object resultLocker = new object();
670    private void ClonedAlgorithm_Started(object sender, EventArgs e) {
671      IAlgorithm algorithm = sender as IAlgorithm;
672      lock (resultLocker) {
673        if (algorithm != null && !results.ContainsKey(algorithm.Name))
674          results.Add(new Result(algorithm.Name, "Contains results for the specific fold.", algorithm.Results));
675      }
676    }
677
678    private void ClonedAlgorithm_Paused(object sender, EventArgs e) {
679      lock (locker) {
680        if (pausePending && clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started))
681          OnPaused();
682      }
683    }
684
685    private void ClonedAlgorithm_Stopped(object sender, EventArgs e) {
686      lock (locker) {
687        if (!stopPending && ExecutionState == ExecutionState.Started) {
688          IAlgorithm preparedAlgorithm = clonedAlgorithms.FirstOrDefault(alg => alg.ExecutionState == ExecutionState.Prepared ||
689                                                                                alg.ExecutionState == ExecutionState.Paused);
690          if (preparedAlgorithm != null) preparedAlgorithm.Start();
691        }
692        if (ExecutionState != ExecutionState.Stopped) {
693          if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped))
694            OnStopped();
695          else if (stopPending &&
696                   clonedAlgorithms.All(
697                     alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped))
698            OnStopped();
699        }
700      }
701    }
702    #endregion
703    #endregion
704
705    #region event firing
706    public event EventHandler ExecutionStateChanged;
707    private void OnExecutionStateChanged() {
708      EventHandler handler = ExecutionStateChanged;
709      if (handler != null) handler(this, EventArgs.Empty);
710    }
711    public event EventHandler ExecutionTimeChanged;
712    private void OnExecutionTimeChanged() {
713      EventHandler handler = ExecutionTimeChanged;
714      if (handler != null) handler(this, EventArgs.Empty);
715    }
716    public event EventHandler Prepared;
717    private void OnPrepared() {
718      ExecutionState = ExecutionState.Prepared;
719      EventHandler handler = Prepared;
720      if (handler != null) handler(this, EventArgs.Empty);
721      OnExecutionTimeChanged();
722    }
723    public event EventHandler Started;
724    private void OnStarted() {
725      ExecutionState = ExecutionState.Started;
726      EventHandler handler = Started;
727      if (handler != null) handler(this, EventArgs.Empty);
728    }
729    public event EventHandler Paused;
730    private void OnPaused() {
731      pausePending = false;
732      ExecutionState = ExecutionState.Paused;
733      EventHandler handler = Paused;
734      if (handler != null) handler(this, EventArgs.Empty);
735    }
736    public event EventHandler Stopped;
737    private void OnStopped() {
738      stopPending = false;
739      Dictionary<string, IItem> collectedResults = new Dictionary<string, IItem>();
740      AggregateResultValues(collectedResults);
741      results.AddRange(collectedResults.Select(x => new Result(x.Key, x.Value)).Cast<IResult>().ToArray());
742      clonedAlgorithms.Clear();
743      runsCounter++;
744      runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
745      ExecutionState = ExecutionState.Stopped;
746      EventHandler handler = Stopped;
747      if (handler != null) handler(this, EventArgs.Empty);
748    }
749    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
750    private void OnExceptionOccurred(Exception exception) {
751      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
752      if (handler != null) handler(this, new EventArgs<Exception>(exception));
753    }
754    public event EventHandler StoreAlgorithmInEachRunChanged;
755    private void OnStoreAlgorithmInEachRunChanged() {
756      EventHandler handler = StoreAlgorithmInEachRunChanged;
757      if (handler != null) handler(this, EventArgs.Empty);
758    }
759    #endregion
760  }
761}
Note: See TracBrowser for help on using the repository browser.