Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs @ 17094

Last change on this file since 17094 was 16661, checked in by gkronber, 6 years ago

#2933: use RandomSeedGenerator in CrossValidation

File size: 34.9 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Drawing;
25using System.Linq;
26using System.Threading;
27using System.Threading.Tasks;
28using HeuristicLab.Collections;
29using HeuristicLab.Common;
30using HeuristicLab.Core;
31using HeuristicLab.Data;
32using HeuristicLab.Optimization;
33using HEAL.Attic;
34using HeuristicLab.Problems.DataAnalysis;
35using HeuristicLab.Problems.DataAnalysis.Symbolic;
36using HeuristicLab.Random;
37
38namespace HeuristicLab.Algorithms.DataAnalysis {
39  [Item("Cross Validation (CV)", "Cross-validation wrapper for data analysis algorithms.")]
40  [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 100)]
41  [StorableType("1C622121-AE5B-42FD-831C-FCA8F8E0AF8D")]
42  public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent {
43    [Storable]
44    private int seed;
45
46    private SemaphoreSlim availableWorkers; // limits the number of concurrent algorithm executions
47    private ManualResetEventSlim allAlgorithmsFinished; // this indicates that all started algorithms have been paused or stopped
48
49    public CrossValidation()
50      : base() {
51      name = ItemName;
52      description = ItemDescription;
53
54      executionState = ExecutionState.Stopped;
55      runs = new RunCollection { OptimizerName = name };
56      runsCounter = 0;
57
58      algorithm = null;
59      clonedAlgorithms = new ItemCollection<IAlgorithm>();
60      results = new ResultCollection();
61
62      folds = new IntValue(2);
63      numberOfWorkers = new IntValue(1);
64      samplesStart = new IntValue(0);
65      samplesEnd = new IntValue(0);
66      shuffleSamples = new BoolValue(false);
67      storeAlgorithmInEachRun = false;
68
69      RegisterEvents();
70      if (Algorithm != null) RegisterAlgorithmEvents();
71    }
72
73    public string Filename { get; set; }
74
75    #region persistence and cloning
76    [StorableConstructor]
77    private CrossValidation(StorableConstructorFlag _) : base(_) {
78    }
79    [StorableHook(HookType.AfterDeserialization)]
80    private void AfterDeserialization() {
81      // BackwardsCompatibility3.3
82      #region Backwards compatible code, remove with 3.4
83      if (shuffleSamples == null) shuffleSamples = new BoolValue(false);
84      #endregion
85
86      RegisterEvents();
87      if (Algorithm != null) RegisterAlgorithmEvents();
88    }
89
90    private CrossValidation(CrossValidation original, Cloner cloner)
91      : base(original, cloner) {
92      executionState = original.executionState;
93      storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
94      runs = cloner.Clone(original.runs);
95      runsCounter = original.runsCounter;
96      algorithm = cloner.Clone(original.algorithm);
97      clonedAlgorithms = cloner.Clone(original.clonedAlgorithms);
98      results = cloner.Clone(original.results);
99
100      folds = cloner.Clone(original.folds);
101      numberOfWorkers = cloner.Clone(original.numberOfWorkers);
102      samplesStart = cloner.Clone(original.samplesStart);
103      samplesEnd = cloner.Clone(original.samplesEnd);
104      shuffleSamples = cloner.Clone(original.shuffleSamples);
105      seed = original.seed;
106
107      RegisterEvents();
108      if (Algorithm != null) RegisterAlgorithmEvents();
109    }
110    public override IDeepCloneable Clone(Cloner cloner) {
111      return new CrossValidation(this, cloner);
112    }
113
114    #endregion
115
116    #region properties
117    [Storable]
118    private IAlgorithm algorithm;
119    public IAlgorithm Algorithm {
120      get { return algorithm; }
121      set {
122        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
123          throw new InvalidOperationException("Changing the algorithm is only allowed if the CrossValidation is stopped or prepared.");
124        if (algorithm != value) {
125          if (value != null && value.Problem != null && !(value.Problem is IDataAnalysisProblem))
126            throw new ArgumentException("Only algorithms with a DataAnalysisProblem could be used for the cross validation.");
127          if (algorithm != null) DeregisterAlgorithmEvents();
128          algorithm = value;
129          Parameters.Clear();
130
131          if (algorithm != null) {
132            algorithm.StoreAlgorithmInEachRun = false;
133            RegisterAlgorithmEvents();
134            algorithm.Prepare(true);
135            Parameters.AddRange(algorithm.Parameters);
136          }
137          OnAlgorithmChanged();
138          Prepare();
139        }
140      }
141    }
142
143
144    [Storable]
145    private IDataAnalysisProblem problem;
146    public IDataAnalysisProblem Problem {
147      get {
148        if (algorithm == null)
149          return null;
150        return (IDataAnalysisProblem)algorithm.Problem;
151      }
152      set {
153        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
154          throw new InvalidOperationException("Changing the problem is only allowed if the CrossValidation is stopped or prepared.");
155        if (algorithm == null) throw new ArgumentNullException("Could not set a problem before an algorithm was set.");
156        algorithm.Problem = value;
157        problem = value;
158      }
159    }
160
161    IProblem IAlgorithm.Problem {
162      get { return Problem; }
163      set {
164        if (value != null && !ProblemType.IsInstanceOfType(value))
165          throw new ArgumentException("Only DataAnalysisProblems could be used for the cross validation.");
166        Problem = (IDataAnalysisProblem)value;
167      }
168    }
169    public Type ProblemType {
170      get { return typeof(IDataAnalysisProblem); }
171    }
172
173    [Storable]
174    private ItemCollection<IAlgorithm> clonedAlgorithms;
175
176    public IEnumerable<IOptimizer> NestedOptimizers {
177      get {
178        if (Algorithm == null) yield break;
179        yield return Algorithm;
180      }
181    }
182
183    [Storable]
184    private ResultCollection results;
185    public ResultCollection Results {
186      get { return results; }
187    }
188    [Storable]
189    private BoolValue shuffleSamples;
190    public BoolValue ShuffleSamples {
191      get { return shuffleSamples; }
192    }
193    [Storable]
194    private IntValue folds;
195    public IntValue Folds {
196      get { return folds; }
197    }
198    [Storable]
199    private IntValue samplesStart;
200    public IntValue SamplesStart {
201      get { return samplesStart; }
202    }
203    [Storable]
204    private IntValue samplesEnd;
205    public IntValue SamplesEnd {
206      get { return samplesEnd; }
207    }
208    [Storable]
209    private IntValue numberOfWorkers;
210    public IntValue NumberOfWorkers {
211      get { return numberOfWorkers; }
212    }
213
214    [Storable]
215    private bool storeAlgorithmInEachRun;
216    public bool StoreAlgorithmInEachRun {
217      get { return storeAlgorithmInEachRun; }
218      set {
219        if (storeAlgorithmInEachRun != value) {
220          storeAlgorithmInEachRun = value;
221          OnStoreAlgorithmInEachRunChanged();
222        }
223      }
224    }
225
226    [Storable]
227    private int runsCounter;
228    [Storable]
229    private RunCollection runs;
230    public RunCollection Runs {
231      get { return runs; }
232    }
233
234    [Storable]
235    private ExecutionState executionState;
236    public ExecutionState ExecutionState {
237      get { return executionState; }
238      private set {
239        if (executionState != value) {
240          executionState = value;
241          OnExecutionStateChanged();
242          OnItemImageChanged();
243        }
244      }
245    }
246    public static new Image StaticItemImage {
247      get { return HeuristicLab.Common.Resources.VSImageLibrary.Event; }
248    }
249    public override Image ItemImage {
250      get {
251        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
252        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
253        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
254        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
255        else return base.ItemImage;
256      }
257    }
258
259    public TimeSpan ExecutionTime {
260      get {
261        if (ExecutionState != ExecutionState.Prepared)
262          return TimeSpan.FromMilliseconds(clonedAlgorithms.Select(x => x.ExecutionTime.TotalMilliseconds).Sum());
263        return TimeSpan.Zero;
264      }
265    }
266    #endregion
267
268    protected override void OnNameChanged() {
269      base.OnNameChanged();
270      Runs.OptimizerName = Name;
271    }
272
273    public void Prepare() {
274      if (startPending) return;
275      if (ExecutionState == ExecutionState.Started)
276        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
277      results.Clear();
278      clonedAlgorithms.Clear();
279      if (Algorithm != null) {
280        Algorithm.Prepare();
281        if (Algorithm.ExecutionState == ExecutionState.Prepared) OnPrepared();
282      }
283    }
284    public void Prepare(bool clearRuns) {
285      if (clearRuns) runs.Clear();
286      Prepare();
287    }
288
289    private bool startPending;
290    public void Start() {
291      Start(CancellationToken.None);
292    }
293    public void Start(CancellationToken cancellationToken) {
294      lock (locker) {
295        if (startPending) return;
296        startPending = true;
297      }
298
299      try {
300        if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
301          throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));
302        seed = RandomSeedGenerator.GetSeed();
303
304        if (Algorithm == null) return;
305        //create cloned algorithms
306        if (clonedAlgorithms.Count == 0) {
307          int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / Folds.Value;
308          IDataset shuffledDataset = null;
309          for (int i = 0; i < Folds.Value; i++) {
310            var cloner = new Cloner();
311            if (ShuffleSamples.Value) {
312              var random = new FastRandom(seed);
313              var dataAnalysisProblem = (IDataAnalysisProblem)algorithm.Problem;
314              var dataset = (Dataset)dataAnalysisProblem.ProblemData.Dataset;
315              shuffledDataset = shuffledDataset ?? dataset.Shuffle(random);
316              cloner.RegisterClonedObject(dataset, shuffledDataset);
317            }
318            IAlgorithm clonedAlgorithm = cloner.Clone(Algorithm);
319            clonedAlgorithm.Name = algorithm.Name + " Fold " + i;
320            IDataAnalysisProblem problem = clonedAlgorithm.Problem as IDataAnalysisProblem;
321            ISymbolicDataAnalysisProblem symbolicProblem = problem as ISymbolicDataAnalysisProblem;
322
323            int testStart = (i * testSamplesCount) + SamplesStart.Value;
324            int testEnd = (i + 1) == Folds.Value ? SamplesEnd.Value : (i + 1) * testSamplesCount + SamplesStart.Value;
325
326            problem.ProblemData.TrainingPartition.Start = SamplesStart.Value;
327            problem.ProblemData.TrainingPartition.End = SamplesEnd.Value;
328            problem.ProblemData.TestPartition.Start = testStart;
329            problem.ProblemData.TestPartition.End = testEnd;
330            DataAnalysisProblemData problemData = problem.ProblemData as DataAnalysisProblemData;
331            if (problemData != null) {
332              problemData.TrainingPartitionParameter.Hidden = false;
333              problemData.TestPartitionParameter.Hidden = false;
334            }
335
336            if (symbolicProblem != null) {
337              symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
338              symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
339            }
340            clonedAlgorithm.Prepare();
341            clonedAlgorithms.Add(clonedAlgorithm);
342          }
343        }
344
345        OnStarted();
346      } finally {
347        if (startPending) startPending = false;
348      }
349
350      availableWorkers = new SemaphoreSlim(NumberOfWorkers.Value, NumberOfWorkers.Value);
351      allAlgorithmsFinished = new ManualResetEventSlim(false);
352
353      var startedTasks = new List<Task>(clonedAlgorithms.Count);
354
355      //start prepared or paused cloned algorithms
356      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
357        if (pausePending || stopPending || ExecutionState != ExecutionState.Started) break;
358        if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared ||
359            clonedAlgorithm.ExecutionState == ExecutionState.Paused) {
360          availableWorkers.Wait();
361          lock (locker) {
362            if (pausePending || stopPending || ExecutionState != ExecutionState.Started) break;
363            var task = clonedAlgorithm.StartAsync(cancellationToken);
364            startedTasks.Add(task);
365          }
366        }
367      }
368
369      allAlgorithmsFinished.Wait();
370
371      Task.WaitAll(startedTasks.ToArray()); // to get exceptions not handled within the tasks
372    }
373
374    public async Task StartAsync() { await StartAsync(CancellationToken.None); }
375    public async Task StartAsync(CancellationToken cancellationToken) {
376      await AsyncHelper.DoAsync(Start, cancellationToken);
377    }
378
379    private bool pausePending;
380    public void Pause() {
381      if (startPending) return;
382      if (ExecutionState != ExecutionState.Started)
383        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
384      if (!pausePending) {
385        pausePending = true;
386        lock (locker) {
387          var toPause = clonedAlgorithms.Where(x => x.ExecutionState == ExecutionState.Started).ToList();
388          foreach (var optimizer in toPause) {
389            // a race-condition may occur when the optimizer has changed the state by itself in the meantime
390            try { optimizer.Pause(); } catch (InvalidOperationException) { }
391          }
392        }
393      }
394    }
395
396    private bool stopPending;
397    public void Stop() {
398      if (startPending) return;
399      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
400        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".",
401                                                          ExecutionState));
402      if (!stopPending) {
403        stopPending = true;
404        lock (locker) {
405          var toStop = clonedAlgorithms.Where(x => x.ExecutionState == ExecutionState.Started || x.ExecutionState == ExecutionState.Paused).ToList();
406          foreach (var optimizer in toStop) {
407            // a race-condition may occur when the optimizer has changed the state by itself in the meantime
408            try { optimizer.Stop(); } catch (InvalidOperationException) { }
409          }
410        }
411      }
412    }
413
414    #region collect parameters and results
415    public override void CollectParameterValues(IDictionary<string, IItem> values) {
416      values.Add("Algorithm Name", new StringValue(Name));
417      values.Add("Algorithm Type", new StringValue(GetType().GetPrettyName()));
418      values.Add("Folds", new IntValue(Folds.Value));
419
420      if (algorithm != null) {
421        values.Add("CrossValidation Algorithm Name", new StringValue(Algorithm.Name));
422        values.Add("CrossValidation Algorithm Type", new StringValue(Algorithm.GetType().GetPrettyName()));
423        base.CollectParameterValues(values);
424      }
425      if (Problem != null) {
426        values.Add("Problem Name", new StringValue(Problem.Name));
427        values.Add("Problem Type", new StringValue(Problem.GetType().GetPrettyName()));
428        Problem.CollectParameterValues(values);
429      }
430    }
431
432    public void CollectResultValues(IDictionary<string, IItem> results) {
433      var clonedResults = (ResultCollection)this.results.Clone();
434      foreach (var result in clonedResults) {
435        results.Add(result.Name, result.Value);
436      }
437    }
438
439    private void AggregateResultValues(IDictionary<string, IItem> results) {
440      IEnumerable<IRun> runs = clonedAlgorithms.Select(alg => alg.Runs.FirstOrDefault()).Where(run => run != null);
441      IEnumerable<KeyValuePair<string, IItem>> resultCollections = runs.Where(x => x != null).SelectMany(x => x.Results).ToList();
442
443      foreach (IResult result in ExtractAndAggregateResults<IntValue>(resultCollections))
444        results.Add(result.Name, result.Value);
445      foreach (IResult result in ExtractAndAggregateResults<DoubleValue>(resultCollections))
446        results.Add(result.Name, result.Value);
447      foreach (IResult result in ExtractAndAggregateResults<PercentValue>(resultCollections))
448        results.Add(result.Name, result.Value);
449      foreach (IResult result in ExtractAndAggregateRegressionSolutions(resultCollections)) {
450        results.Add(result.Name, result.Value);
451      }
452      foreach (IResult result in ExtractAndAggregateClassificationSolutions(resultCollections)) {
453        results.Add(result.Name, result.Value);
454      }
455      results.Add("Execution Time", new TimeSpanValue(this.ExecutionTime));
456      results.Add("CrossValidation Folds", new RunCollection(runs));
457    }
458
459    private IEnumerable<IResult> ExtractAndAggregateRegressionSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
460      Dictionary<string, List<IRegressionSolution>> resultSolutions = new Dictionary<string, List<IRegressionSolution>>();
461      foreach (var result in resultCollections) {
462        var regressionSolution = result.Value as IRegressionSolution;
463        if (regressionSolution != null) {
464          if (resultSolutions.ContainsKey(result.Key)) {
465            resultSolutions[result.Key].Add(regressionSolution);
466          } else {
467            resultSolutions.Add(result.Key, new List<IRegressionSolution>() { regressionSolution });
468          }
469        }
470      }
471      List<IResult> aggregatedResults = new List<IResult>();
472      foreach (KeyValuePair<string, List<IRegressionSolution>> solutions in resultSolutions) {
473        // clone manually to correctly clone references between cloned root objects
474        Cloner cloner = new Cloner();
475        if (ShuffleSamples.Value) {
476          var dataset = (Dataset)Problem.ProblemData.Dataset;
477          var random = new FastRandom(seed);
478          var shuffledDataset = dataset.Shuffle(random);
479          cloner.RegisterClonedObject(dataset, shuffledDataset);
480        }
481        var problemDataClone = (IRegressionProblemData)cloner.Clone(Problem.ProblemData);
482        // set partitions of problem data clone correctly
483        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
484        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
485        // clone models
486        var ensembleSolution = new RegressionEnsembleSolution(problemDataClone);
487        ensembleSolution.AddRegressionSolutions(solutions.Value);
488
489        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
490      }
491      List<IResult> flattenedResults = new List<IResult>();
492      CollectResultsRecursively("", aggregatedResults, flattenedResults);
493      return flattenedResults;
494    }
495
496    private IEnumerable<IResult> ExtractAndAggregateClassificationSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
497      Dictionary<string, List<IClassificationSolution>> resultSolutions = new Dictionary<string, List<IClassificationSolution>>();
498      foreach (var result in resultCollections) {
499        var classificationSolution = result.Value as IClassificationSolution;
500        if (classificationSolution != null) {
501          if (resultSolutions.ContainsKey(result.Key)) {
502            resultSolutions[result.Key].Add(classificationSolution);
503          } else {
504            resultSolutions.Add(result.Key, new List<IClassificationSolution>() { classificationSolution });
505          }
506        }
507      }
508      var aggregatedResults = new List<IResult>();
509      foreach (KeyValuePair<string, List<IClassificationSolution>> solutions in resultSolutions) {
510        // at least one algorithm (GBT with logistic regression loss) produces a classification solution even though the original problem is a regression problem.
511        var targetVariable = solutions.Value.First().ProblemData.TargetVariable;
512        var dataset = (Dataset)Problem.ProblemData.Dataset;
513        if (ShuffleSamples.Value) {
514          var random = new FastRandom(seed);
515          dataset = dataset.Shuffle(random);
516        }
517        var problemDataClone = new ClassificationProblemData(dataset, Problem.ProblemData.AllowedInputVariables, targetVariable);
518        // set partitions of problem data clone correctly
519        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
520        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
521        // clone models
522        var ensembleSolution = new ClassificationEnsembleSolution(problemDataClone);
523        ensembleSolution.AddClassificationSolutions(solutions.Value);
524
525        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
526      }
527      List<IResult> flattenedResults = new List<IResult>();
528      CollectResultsRecursively("", aggregatedResults, flattenedResults);
529      return flattenedResults;
530    }
531
532    private void CollectResultsRecursively(string path, IEnumerable<IResult> results, IList<IResult> flattenedResults) {
533      foreach (IResult result in results) {
534        flattenedResults.Add(new Result(path + result.Name, result.Value));
535        ResultCollection childCollection = result.Value as ResultCollection;
536        if (childCollection != null) {
537          CollectResultsRecursively(path + result.Name + ".", childCollection, flattenedResults);
538        }
539      }
540    }
541
542    private static IEnumerable<IResult> ExtractAndAggregateResults<T>(IEnumerable<KeyValuePair<string, IItem>> results)
543  where T : class, IItem, new() {
544      Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
545      foreach (var resultValue in results.Where(r => r.Value.GetType() == typeof(T))) {
546        if (!resultValues.ContainsKey(resultValue.Key))
547          resultValues[resultValue.Key] = new List<double>();
548        resultValues[resultValue.Key].Add(ConvertToDouble(resultValue.Value));
549      }
550
551      DoubleValue doubleValue;
552      if (typeof(T) == typeof(PercentValue))
553        doubleValue = new PercentValue();
554      else if (typeof(T) == typeof(DoubleValue))
555        doubleValue = new DoubleValue();
556      else if (typeof(T) == typeof(IntValue))
557        doubleValue = new DoubleValue();
558      else
559        throw new NotSupportedException();
560
561      List<IResult> aggregatedResults = new List<IResult>();
562      foreach (KeyValuePair<string, List<double>> resultValue in resultValues) {
563        doubleValue.Value = resultValue.Value.Average();
564        aggregatedResults.Add(new Result(resultValue.Key + " (average)", (IItem)doubleValue.Clone()));
565        doubleValue.Value = resultValue.Value.StandardDeviation();
566        aggregatedResults.Add(new Result(resultValue.Key + " (std.dev.)", (IItem)doubleValue.Clone()));
567      }
568      return aggregatedResults;
569    }
570
571    private static double ConvertToDouble(IItem item) {
572      if (item is DoubleValue) return ((DoubleValue)item).Value;
573      else if (item is IntValue) return ((IntValue)item).Value;
574      else throw new NotSupportedException("Could not convert any item type to double");
575    }
576    #endregion
577
578    #region events
579    private void RegisterEvents() {
580      Folds.ValueChanged += new EventHandler(Folds_ValueChanged);
581      RegisterClonedAlgorithmsEvents();
582    }
583    private void Folds_ValueChanged(object sender, EventArgs e) {
584      if (ExecutionState != ExecutionState.Prepared)
585        throw new InvalidOperationException("Can not change number of folds if the execution state is not prepared.");
586    }
587
588
589    #region template algorithms events
590    public event EventHandler AlgorithmChanged;
591    private void OnAlgorithmChanged() {
592      EventHandler handler = AlgorithmChanged;
593      if (handler != null) handler(this, EventArgs.Empty);
594      OnProblemChanged();
595      if (Problem == null) ExecutionState = ExecutionState.Stopped;
596    }
597    private void RegisterAlgorithmEvents() {
598      algorithm.ProblemChanged += new EventHandler(Algorithm_ProblemChanged);
599      algorithm.ExecutionStateChanged += new EventHandler(Algorithm_ExecutionStateChanged);
600      if (Problem != null) {
601        Problem.Reset += new EventHandler(Problem_Reset);
602      }
603    }
604    private void DeregisterAlgorithmEvents() {
605      algorithm.ProblemChanged -= new EventHandler(Algorithm_ProblemChanged);
606      algorithm.ExecutionStateChanged -= new EventHandler(Algorithm_ExecutionStateChanged);
607      if (Problem != null) {
608        Problem.Reset -= new EventHandler(Problem_Reset);
609      }
610    }
611    private void Algorithm_ProblemChanged(object sender, EventArgs e) {
612      if (algorithm.Problem != null && !(algorithm.Problem is IDataAnalysisProblem)) {
613        algorithm.Problem = problem;
614        throw new ArgumentException("A cross validation algorithm can only contain DataAnalysisProblems.");
615      }
616      if (problem != null) problem.Reset -= new EventHandler(Problem_Reset);
617      problem = (IDataAnalysisProblem)algorithm.Problem;
618      if (problem != null) problem.Reset += new EventHandler(Problem_Reset);
619      OnProblemChanged();
620    }
621    public event EventHandler ProblemChanged;
622    private void OnProblemChanged() {
623      EventHandler handler = ProblemChanged;
624      if (handler != null) handler(this, EventArgs.Empty);
625      ConfigureProblem();
626    }
627    private void Problem_Reset(object sender, EventArgs e) {
628      ConfigureProblem();
629    }
630    private void ConfigureProblem() {
631      SamplesStart.Value = 0;
632      if (Problem != null) {
633        SamplesEnd.Value = Problem.ProblemData.Dataset.Rows;
634
635        DataAnalysisProblemData problemData = Problem.ProblemData as DataAnalysisProblemData;
636        if (problemData != null) {
637          problemData.TrainingPartitionParameter.Hidden = true;
638          problemData.TestPartitionParameter.Hidden = true;
639        }
640        ISymbolicDataAnalysisProblem symbolicProblem = Problem as ISymbolicDataAnalysisProblem;
641        if (symbolicProblem != null) {
642          symbolicProblem.FitnessCalculationPartitionParameter.Hidden = true;
643          symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
644          symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
645          symbolicProblem.ValidationPartitionParameter.Hidden = true;
646          symbolicProblem.ValidationPartition.Start = 0;
647          symbolicProblem.ValidationPartition.End = 0;
648        }
649      } else
650        SamplesEnd.Value = 0;
651    }
652
653    private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
654      switch (Algorithm.ExecutionState) {
655        case ExecutionState.Prepared:
656          OnPrepared();
657          break;
658        case ExecutionState.Started: throw new InvalidOperationException("Algorithm template can not be started.");
659        case ExecutionState.Paused: throw new InvalidOperationException("Algorithm template can not be paused.");
660        case ExecutionState.Stopped:
661          OnStopped();
662          break;
663      }
664    }
665    #endregion
666
667    #region clonedAlgorithms events
668    private void RegisterClonedAlgorithmsEvents() {
669      clonedAlgorithms.ItemsAdded += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
670      clonedAlgorithms.ItemsRemoved += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
671      clonedAlgorithms.CollectionReset += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
672      foreach (IAlgorithm algorithm in clonedAlgorithms)
673        RegisterClonedAlgorithmEvents(algorithm);
674    }
675    private void DeregisterClonedAlgorithmsEvents() {
676      clonedAlgorithms.ItemsAdded -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
677      clonedAlgorithms.ItemsRemoved -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
678      clonedAlgorithms.CollectionReset -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
679      foreach (IAlgorithm algorithm in clonedAlgorithms)
680        DeregisterClonedAlgorithmEvents(algorithm);
681    }
682    private void ClonedAlgorithms_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
683      foreach (IAlgorithm algorithm in e.Items)
684        RegisterClonedAlgorithmEvents(algorithm);
685    }
686    private void ClonedAlgorithms_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
687      foreach (IAlgorithm algorithm in e.Items)
688        DeregisterClonedAlgorithmEvents(algorithm);
689    }
690    private void ClonedAlgorithms_CollectionReset(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
691      foreach (IAlgorithm algorithm in e.OldItems)
692        DeregisterClonedAlgorithmEvents(algorithm);
693      foreach (IAlgorithm algorithm in e.Items)
694        RegisterClonedAlgorithmEvents(algorithm);
695    }
696    private void RegisterClonedAlgorithmEvents(IAlgorithm algorithm) {
697      algorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
698      algorithm.ExecutionTimeChanged += new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
699      algorithm.Started += new EventHandler(ClonedAlgorithm_Started);
700      algorithm.Paused += new EventHandler(ClonedAlgorithm_Paused);
701      algorithm.Stopped += new EventHandler(ClonedAlgorithm_Stopped);
702    }
703    private void DeregisterClonedAlgorithmEvents(IAlgorithm algorithm) {
704      algorithm.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
705      algorithm.ExecutionTimeChanged -= new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
706      algorithm.Started -= new EventHandler(ClonedAlgorithm_Started);
707      algorithm.Paused -= new EventHandler(ClonedAlgorithm_Paused);
708      algorithm.Stopped -= new EventHandler(ClonedAlgorithm_Stopped);
709    }
710    private void ClonedAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
711      Pause();
712      OnExceptionOccurred(e.Value);
713    }
714    private void ClonedAlgorithm_ExecutionTimeChanged(object sender, EventArgs e) {
715      OnExecutionTimeChanged();
716    }
717
718    private readonly object locker = new object();
719    private readonly object resultLocker = new object();
720    private void ClonedAlgorithm_Started(object sender, EventArgs e) {
721      IAlgorithm algorithm = sender as IAlgorithm;
722      lock (resultLocker) {
723        if (algorithm != null && !results.ContainsKey(algorithm.Name))
724          results.Add(new Result(algorithm.Name, "Contains results for the specific fold.", algorithm.Results));
725      }
726    }
727
728    private void ClonedAlgorithm_Paused(object sender, EventArgs e) {
729      lock (locker) {
730        availableWorkers.Release();
731        if (clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started)) {
732          OnPaused();
733          allAlgorithmsFinished.Set();
734        }
735      }
736    }
737
738    private void ClonedAlgorithm_Stopped(object sender, EventArgs e) {
739      lock (locker) {
740        // if the algorithm was in paused state, its worker has already been released
741        if (availableWorkers.CurrentCount < NumberOfWorkers.Value)
742          availableWorkers.Release();
743        if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped)) {
744          OnStopped();
745          allAlgorithmsFinished.Set();
746        } else if (stopPending && clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped)) {
747          OnStopped();
748          allAlgorithmsFinished.Set();
749        }
750      }
751    }
752    #endregion
753    #endregion
754
755    #region event firing
756    public event EventHandler ExecutionStateChanged;
757    private void OnExecutionStateChanged() {
758      EventHandler handler = ExecutionStateChanged;
759      if (handler != null) handler(this, EventArgs.Empty);
760    }
761    public event EventHandler ExecutionTimeChanged;
762    private void OnExecutionTimeChanged() {
763      EventHandler handler = ExecutionTimeChanged;
764      if (handler != null) handler(this, EventArgs.Empty);
765    }
766    public event EventHandler Prepared;
767    private void OnPrepared() {
768      ExecutionState = ExecutionState.Prepared;
769      EventHandler handler = Prepared;
770      if (handler != null) handler(this, EventArgs.Empty);
771      OnExecutionTimeChanged();
772    }
773    public event EventHandler Started;
774    private void OnStarted() {
775      startPending = false;
776      ExecutionState = ExecutionState.Started;
777      EventHandler handler = Started;
778      if (handler != null) handler(this, EventArgs.Empty);
779    }
780    public event EventHandler Paused;
781    private void OnPaused() {
782      pausePending = false;
783      ExecutionState = ExecutionState.Paused;
784      EventHandler handler = Paused;
785      if (handler != null) handler(this, EventArgs.Empty);
786    }
787    public event EventHandler Stopped;
788    private void OnStopped() {
789      stopPending = false;
790      Dictionary<string, IItem> collectedResults = new Dictionary<string, IItem>();
791      AggregateResultValues(collectedResults);
792      results.AddRange(collectedResults.Select(x => new Result(x.Key, x.Value)).Cast<IResult>().ToArray());
793      clonedAlgorithms.Clear();
794      runsCounter++;
795      runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
796      ExecutionState = ExecutionState.Stopped;
797      EventHandler handler = Stopped;
798      if (handler != null) handler(this, EventArgs.Empty);
799    }
800    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
801    private void OnExceptionOccurred(Exception exception) {
802      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
803      if (handler != null) handler(this, new EventArgs<Exception>(exception));
804    }
805    public event EventHandler StoreAlgorithmInEachRunChanged;
806    private void OnStoreAlgorithmInEachRunChanged() {
807      EventHandler handler = StoreAlgorithmInEachRunChanged;
808      if (handler != null) handler(this, EventArgs.Empty);
809    }
810    #endregion
811  }
812}
Note: See TracBrowser for help on using the repository browser.