source: stable/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs @ 16167

Last change on this file since 16167 was 15584, checked in by swagner, 3 years ago

#2640: Updated year of copyrights in license headers on stable

File size: 34.9 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Drawing;
25using System.Linq;
26using System.Threading;
27using System.Threading.Tasks;
28using HeuristicLab.Collections;
29using HeuristicLab.Common;
30using HeuristicLab.Core;
31using HeuristicLab.Data;
32using HeuristicLab.Optimization;
33using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
34using HeuristicLab.Problems.DataAnalysis;
35using HeuristicLab.Problems.DataAnalysis.Symbolic;
36using HeuristicLab.Random;
37
38namespace HeuristicLab.Algorithms.DataAnalysis {
39  [Item("Cross Validation (CV)", "Cross-validation wrapper for data analysis algorithms.")]
40  [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 100)]
41  [StorableClass]
42  public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent {
43    [Storable]
44    private int seed;
45
46    private SemaphoreSlim availableWorkers; // limits the number of concurrent algorithm executions
47    private ManualResetEventSlim allAlgorithmsFinished; // this indicates that all started algorithms have been paused or stopped
48
49    public CrossValidation()
50      : base() {
51      name = ItemName;
52      description = ItemDescription;
53
54      executionState = ExecutionState.Stopped;
55      runs = new RunCollection { OptimizerName = name };
56      runsCounter = 0;
57
58      algorithm = null;
59      clonedAlgorithms = new ItemCollection<IAlgorithm>();
60      results = new ResultCollection();
61
62      folds = new IntValue(2);
63      numberOfWorkers = new IntValue(1);
64      samplesStart = new IntValue(0);
65      samplesEnd = new IntValue(0);
66      shuffleSamples = new BoolValue(false);
67      storeAlgorithmInEachRun = false;
68
69      RegisterEvents();
70      if (Algorithm != null) RegisterAlgorithmEvents();
71    }
72
73    public string Filename { get; set; }
74
75    #region persistence and cloning
76    [StorableConstructor]
77    private CrossValidation(bool deserializing)
78      : base(deserializing) {
79    }
80    [StorableHook(HookType.AfterDeserialization)]
81    private void AfterDeserialization() {
82      // BackwardsCompatibility3.3
83      #region Backwards compatible code, remove with 3.4
84      if (shuffleSamples == null) shuffleSamples = new BoolValue(false);
85      #endregion
86
87      RegisterEvents();
88      if (Algorithm != null) RegisterAlgorithmEvents();
89    }
90
91    private CrossValidation(CrossValidation original, Cloner cloner)
92      : base(original, cloner) {
93      executionState = original.executionState;
94      storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
95      runs = cloner.Clone(original.runs);
96      runsCounter = original.runsCounter;
97      algorithm = cloner.Clone(original.algorithm);
98      clonedAlgorithms = cloner.Clone(original.clonedAlgorithms);
99      results = cloner.Clone(original.results);
100
101      folds = cloner.Clone(original.folds);
102      numberOfWorkers = cloner.Clone(original.numberOfWorkers);
103      samplesStart = cloner.Clone(original.samplesStart);
104      samplesEnd = cloner.Clone(original.samplesEnd);
105      shuffleSamples = cloner.Clone(original.shuffleSamples);
106      seed = original.seed;
107
108      RegisterEvents();
109      if (Algorithm != null) RegisterAlgorithmEvents();
110    }
111    public override IDeepCloneable Clone(Cloner cloner) {
112      return new CrossValidation(this, cloner);
113    }
114
115    #endregion
116
117    #region properties
118    [Storable]
119    private IAlgorithm algorithm;
120    public IAlgorithm Algorithm {
121      get { return algorithm; }
122      set {
123        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
124          throw new InvalidOperationException("Changing the algorithm is only allowed if the CrossValidation is stopped or prepared.");
125        if (algorithm != value) {
126          if (value != null && value.Problem != null && !(value.Problem is IDataAnalysisProblem))
127            throw new ArgumentException("Only algorithms with a DataAnalysisProblem could be used for the cross validation.");
128          if (algorithm != null) DeregisterAlgorithmEvents();
129          algorithm = value;
130          Parameters.Clear();
131
132          if (algorithm != null) {
133            algorithm.StoreAlgorithmInEachRun = false;
134            RegisterAlgorithmEvents();
135            algorithm.Prepare(true);
136            Parameters.AddRange(algorithm.Parameters);
137          }
138          OnAlgorithmChanged();
139          Prepare();
140        }
141      }
142    }
143
144
145    [Storable]
146    private IDataAnalysisProblem problem;
147    public IDataAnalysisProblem Problem {
148      get {
149        if (algorithm == null)
150          return null;
151        return (IDataAnalysisProblem)algorithm.Problem;
152      }
153      set {
154        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
155          throw new InvalidOperationException("Changing the problem is only allowed if the CrossValidation is stopped or prepared.");
156        if (algorithm == null) throw new ArgumentNullException("Could not set a problem before an algorithm was set.");
157        algorithm.Problem = value;
158        problem = value;
159      }
160    }
161
162    IProblem IAlgorithm.Problem {
163      get { return Problem; }
164      set {
165        if (value != null && !ProblemType.IsInstanceOfType(value))
166          throw new ArgumentException("Only DataAnalysisProblems could be used for the cross validation.");
167        Problem = (IDataAnalysisProblem)value;
168      }
169    }
170    public Type ProblemType {
171      get { return typeof(IDataAnalysisProblem); }
172    }
173
174    [Storable]
175    private ItemCollection<IAlgorithm> clonedAlgorithms;
176
177    public IEnumerable<IOptimizer> NestedOptimizers {
178      get {
179        if (Algorithm == null) yield break;
180        yield return Algorithm;
181      }
182    }
183
184    [Storable]
185    private ResultCollection results;
186    public ResultCollection Results {
187      get { return results; }
188    }
189    [Storable]
190    private BoolValue shuffleSamples;
191    public BoolValue ShuffleSamples {
192      get { return shuffleSamples; }
193    }
194    [Storable]
195    private IntValue folds;
196    public IntValue Folds {
197      get { return folds; }
198    }
199    [Storable]
200    private IntValue samplesStart;
201    public IntValue SamplesStart {
202      get { return samplesStart; }
203    }
204    [Storable]
205    private IntValue samplesEnd;
206    public IntValue SamplesEnd {
207      get { return samplesEnd; }
208    }
209    [Storable]
210    private IntValue numberOfWorkers;
211    public IntValue NumberOfWorkers {
212      get { return numberOfWorkers; }
213    }
214
215    [Storable]
216    private bool storeAlgorithmInEachRun;
217    public bool StoreAlgorithmInEachRun {
218      get { return storeAlgorithmInEachRun; }
219      set {
220        if (storeAlgorithmInEachRun != value) {
221          storeAlgorithmInEachRun = value;
222          OnStoreAlgorithmInEachRunChanged();
223        }
224      }
225    }
226
227    [Storable]
228    private int runsCounter;
229    [Storable]
230    private RunCollection runs;
231    public RunCollection Runs {
232      get { return runs; }
233    }
234
235    [Storable]
236    private ExecutionState executionState;
237    public ExecutionState ExecutionState {
238      get { return executionState; }
239      private set {
240        if (executionState != value) {
241          executionState = value;
242          OnExecutionStateChanged();
243          OnItemImageChanged();
244        }
245      }
246    }
247    public static new Image StaticItemImage {
248      get { return HeuristicLab.Common.Resources.VSImageLibrary.Event; }
249    }
250    public override Image ItemImage {
251      get {
252        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
253        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
254        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
255        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
256        else return base.ItemImage;
257      }
258    }
259
260    public TimeSpan ExecutionTime {
261      get {
262        if (ExecutionState != ExecutionState.Prepared)
263          return TimeSpan.FromMilliseconds(clonedAlgorithms.Select(x => x.ExecutionTime.TotalMilliseconds).Sum());
264        return TimeSpan.Zero;
265      }
266    }
267    #endregion
268
269    protected override void OnNameChanged() {
270      base.OnNameChanged();
271      Runs.OptimizerName = Name;
272    }
273
274    public void Prepare() {
275      if (startPending) return;
276      if (ExecutionState == ExecutionState.Started)
277        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
278      results.Clear();
279      clonedAlgorithms.Clear();
280      if (Algorithm != null) {
281        Algorithm.Prepare();
282        if (Algorithm.ExecutionState == ExecutionState.Prepared) OnPrepared();
283      }
284    }
285    public void Prepare(bool clearRuns) {
286      if (clearRuns) runs.Clear();
287      Prepare();
288    }
289
290    private bool startPending;
291    public void Start() {
292      Start(CancellationToken.None);
293    }
294    public void Start(CancellationToken cancellationToken) {
295      lock (locker) {
296        if (startPending) return;
297        startPending = true;
298      }
299
300      try {
301        if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
302          throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));
303        seed = new FastRandom().NextInt();
304
305        if (Algorithm == null) return;
306        //create cloned algorithms
307        if (clonedAlgorithms.Count == 0) {
308          int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / Folds.Value;
309          IDataset shuffledDataset = null;
310          for (int i = 0; i < Folds.Value; i++) {
311            var cloner = new Cloner();
312            if (ShuffleSamples.Value) {
313              var random = new FastRandom(seed);
314              var dataAnalysisProblem = (IDataAnalysisProblem)algorithm.Problem;
315              var dataset = (Dataset)dataAnalysisProblem.ProblemData.Dataset;
316              shuffledDataset = shuffledDataset ?? dataset.Shuffle(random);
317              cloner.RegisterClonedObject(dataset, shuffledDataset);
318            }
319            IAlgorithm clonedAlgorithm = cloner.Clone(Algorithm);
320            clonedAlgorithm.Name = algorithm.Name + " Fold " + i;
321            IDataAnalysisProblem problem = clonedAlgorithm.Problem as IDataAnalysisProblem;
322            ISymbolicDataAnalysisProblem symbolicProblem = problem as ISymbolicDataAnalysisProblem;
323
324            int testStart = (i * testSamplesCount) + SamplesStart.Value;
325            int testEnd = (i + 1) == Folds.Value ? SamplesEnd.Value : (i + 1) * testSamplesCount + SamplesStart.Value;
326
327            problem.ProblemData.TrainingPartition.Start = SamplesStart.Value;
328            problem.ProblemData.TrainingPartition.End = SamplesEnd.Value;
329            problem.ProblemData.TestPartition.Start = testStart;
330            problem.ProblemData.TestPartition.End = testEnd;
331            DataAnalysisProblemData problemData = problem.ProblemData as DataAnalysisProblemData;
332            if (problemData != null) {
333              problemData.TrainingPartitionParameter.Hidden = false;
334              problemData.TestPartitionParameter.Hidden = false;
335            }
336
337            if (symbolicProblem != null) {
338              symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
339              symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
340            }
341            clonedAlgorithm.Prepare();
342            clonedAlgorithms.Add(clonedAlgorithm);
343          }
344        }
345
346        OnStarted();
347      } finally {
348        if (startPending) startPending = false;
349      }
350
351      availableWorkers = new SemaphoreSlim(NumberOfWorkers.Value, NumberOfWorkers.Value);
352      allAlgorithmsFinished = new ManualResetEventSlim(false);
353
354      var startedTasks = new List<Task>(clonedAlgorithms.Count);
355
356      //start prepared or paused cloned algorithms
357      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
358        if (pausePending || stopPending || ExecutionState != ExecutionState.Started) break;
359        if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared ||
360            clonedAlgorithm.ExecutionState == ExecutionState.Paused) {
361          availableWorkers.Wait();
362          lock (locker) {
363            if (pausePending || stopPending || ExecutionState != ExecutionState.Started) break;
364            var task = clonedAlgorithm.StartAsync(cancellationToken);
365            startedTasks.Add(task);
366          }
367        }
368      }
369
370      allAlgorithmsFinished.Wait();
371
372      Task.WaitAll(startedTasks.ToArray()); // to get exceptions not handled within the tasks
373    }
374
375    public async Task StartAsync() { await StartAsync(CancellationToken.None); }
376    public async Task StartAsync(CancellationToken cancellationToken) {
377      await AsyncHelper.DoAsync(Start, cancellationToken);
378    }
379
380    private bool pausePending;
381    public void Pause() {
382      if (startPending) return;
383      if (ExecutionState != ExecutionState.Started)
384        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
385      if (!pausePending) {
386        pausePending = true;
387        lock (locker) {
388          var toPause = clonedAlgorithms.Where(x => x.ExecutionState == ExecutionState.Started).ToList();
389          foreach (var optimizer in toPause) {
390            // a race-condition may occur when the optimizer has changed the state by itself in the meantime
391            try { optimizer.Pause(); } catch (InvalidOperationException) { }
392          }
393        }
394      }
395    }
396
397    private bool stopPending;
398    public void Stop() {
399      if (startPending) return;
400      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
401        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".",
402                                                          ExecutionState));
403      if (!stopPending) {
404        stopPending = true;
405        lock (locker) {
406          var toStop = clonedAlgorithms.Where(x => x.ExecutionState == ExecutionState.Started || x.ExecutionState == ExecutionState.Paused).ToList();
407          foreach (var optimizer in toStop) {
408            // a race-condition may occur when the optimizer has changed the state by itself in the meantime
409            try { optimizer.Stop(); } catch (InvalidOperationException) { }
410          }
411        }
412      }
413    }
414
415    #region collect parameters and results
416    public override void CollectParameterValues(IDictionary<string, IItem> values) {
417      values.Add("Algorithm Name", new StringValue(Name));
418      values.Add("Algorithm Type", new StringValue(GetType().GetPrettyName()));
419      values.Add("Folds", new IntValue(Folds.Value));
420
421      if (algorithm != null) {
422        values.Add("CrossValidation Algorithm Name", new StringValue(Algorithm.Name));
423        values.Add("CrossValidation Algorithm Type", new StringValue(Algorithm.GetType().GetPrettyName()));
424        base.CollectParameterValues(values);
425      }
426      if (Problem != null) {
427        values.Add("Problem Name", new StringValue(Problem.Name));
428        values.Add("Problem Type", new StringValue(Problem.GetType().GetPrettyName()));
429        Problem.CollectParameterValues(values);
430      }
431    }
432
433    public void CollectResultValues(IDictionary<string, IItem> results) {
434      var clonedResults = (ResultCollection)this.results.Clone();
435      foreach (var result in clonedResults) {
436        results.Add(result.Name, result.Value);
437      }
438    }
439
440    private void AggregateResultValues(IDictionary<string, IItem> results) {
441      IEnumerable<IRun> runs = clonedAlgorithms.Select(alg => alg.Runs.FirstOrDefault()).Where(run => run != null);
442      IEnumerable<KeyValuePair<string, IItem>> resultCollections = runs.Where(x => x != null).SelectMany(x => x.Results).ToList();
443
444      foreach (IResult result in ExtractAndAggregateResults<IntValue>(resultCollections))
445        results.Add(result.Name, result.Value);
446      foreach (IResult result in ExtractAndAggregateResults<DoubleValue>(resultCollections))
447        results.Add(result.Name, result.Value);
448      foreach (IResult result in ExtractAndAggregateResults<PercentValue>(resultCollections))
449        results.Add(result.Name, result.Value);
450      foreach (IResult result in ExtractAndAggregateRegressionSolutions(resultCollections)) {
451        results.Add(result.Name, result.Value);
452      }
453      foreach (IResult result in ExtractAndAggregateClassificationSolutions(resultCollections)) {
454        results.Add(result.Name, result.Value);
455      }
456      results.Add("Execution Time", new TimeSpanValue(this.ExecutionTime));
457      results.Add("CrossValidation Folds", new RunCollection(runs));
458    }
459
460    private IEnumerable<IResult> ExtractAndAggregateRegressionSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
461      Dictionary<string, List<IRegressionSolution>> resultSolutions = new Dictionary<string, List<IRegressionSolution>>();
462      foreach (var result in resultCollections) {
463        var regressionSolution = result.Value as IRegressionSolution;
464        if (regressionSolution != null) {
465          if (resultSolutions.ContainsKey(result.Key)) {
466            resultSolutions[result.Key].Add(regressionSolution);
467          } else {
468            resultSolutions.Add(result.Key, new List<IRegressionSolution>() { regressionSolution });
469          }
470        }
471      }
472      List<IResult> aggregatedResults = new List<IResult>();
473      foreach (KeyValuePair<string, List<IRegressionSolution>> solutions in resultSolutions) {
474        // clone manually to correctly clone references between cloned root objects
475        Cloner cloner = new Cloner();
476        if (ShuffleSamples.Value) {
477          var dataset = (Dataset)Problem.ProblemData.Dataset;
478          var random = new FastRandom(seed);
479          var shuffledDataset = dataset.Shuffle(random);
480          cloner.RegisterClonedObject(dataset, shuffledDataset);
481        }
482        var problemDataClone = (IRegressionProblemData)cloner.Clone(Problem.ProblemData);
483        // set partitions of problem data clone correctly
484        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
485        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
486        // clone models
487        var ensembleSolution = new RegressionEnsembleSolution(problemDataClone);
488        ensembleSolution.AddRegressionSolutions(solutions.Value);
489
490        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
491      }
492      List<IResult> flattenedResults = new List<IResult>();
493      CollectResultsRecursively("", aggregatedResults, flattenedResults);
494      return flattenedResults;
495    }
496
497    private IEnumerable<IResult> ExtractAndAggregateClassificationSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
498      Dictionary<string, List<IClassificationSolution>> resultSolutions = new Dictionary<string, List<IClassificationSolution>>();
499      foreach (var result in resultCollections) {
500        var classificationSolution = result.Value as IClassificationSolution;
501        if (classificationSolution != null) {
502          if (resultSolutions.ContainsKey(result.Key)) {
503            resultSolutions[result.Key].Add(classificationSolution);
504          } else {
505            resultSolutions.Add(result.Key, new List<IClassificationSolution>() { classificationSolution });
506          }
507        }
508      }
509      var aggregatedResults = new List<IResult>();
510      foreach (KeyValuePair<string, List<IClassificationSolution>> solutions in resultSolutions) {
511        // at least one algorithm (GBT with logistic regression loss) produces a classification solution even though the original problem is a regression problem.
512        var targetVariable = solutions.Value.First().ProblemData.TargetVariable;
513        var dataset = (Dataset)Problem.ProblemData.Dataset;
514        if (ShuffleSamples.Value) {
515          var random = new FastRandom(seed);
516          dataset = dataset.Shuffle(random);
517        }
518        var problemDataClone = new ClassificationProblemData(dataset, Problem.ProblemData.AllowedInputVariables, targetVariable);
519        // set partitions of problem data clone correctly
520        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
521        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
522        // clone models
523        var ensembleSolution = new ClassificationEnsembleSolution(problemDataClone);
524        ensembleSolution.AddClassificationSolutions(solutions.Value);
525
526        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
527      }
528      List<IResult> flattenedResults = new List<IResult>();
529      CollectResultsRecursively("", aggregatedResults, flattenedResults);
530      return flattenedResults;
531    }
532
533    private void CollectResultsRecursively(string path, IEnumerable<IResult> results, IList<IResult> flattenedResults) {
534      foreach (IResult result in results) {
535        flattenedResults.Add(new Result(path + result.Name, result.Value));
536        ResultCollection childCollection = result.Value as ResultCollection;
537        if (childCollection != null) {
538          CollectResultsRecursively(path + result.Name + ".", childCollection, flattenedResults);
539        }
540      }
541    }
542
543    private static IEnumerable<IResult> ExtractAndAggregateResults<T>(IEnumerable<KeyValuePair<string, IItem>> results)
544  where T : class, IItem, new() {
545      Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
546      foreach (var resultValue in results.Where(r => r.Value.GetType() == typeof(T))) {
547        if (!resultValues.ContainsKey(resultValue.Key))
548          resultValues[resultValue.Key] = new List<double>();
549        resultValues[resultValue.Key].Add(ConvertToDouble(resultValue.Value));
550      }
551
552      DoubleValue doubleValue;
553      if (typeof(T) == typeof(PercentValue))
554        doubleValue = new PercentValue();
555      else if (typeof(T) == typeof(DoubleValue))
556        doubleValue = new DoubleValue();
557      else if (typeof(T) == typeof(IntValue))
558        doubleValue = new DoubleValue();
559      else
560        throw new NotSupportedException();
561
562      List<IResult> aggregatedResults = new List<IResult>();
563      foreach (KeyValuePair<string, List<double>> resultValue in resultValues) {
564        doubleValue.Value = resultValue.Value.Average();
565        aggregatedResults.Add(new Result(resultValue.Key + " (average)", (IItem)doubleValue.Clone()));
566        doubleValue.Value = resultValue.Value.StandardDeviation();
567        aggregatedResults.Add(new Result(resultValue.Key + " (std.dev.)", (IItem)doubleValue.Clone()));
568      }
569      return aggregatedResults;
570    }
571
572    private static double ConvertToDouble(IItem item) {
573      if (item is DoubleValue) return ((DoubleValue)item).Value;
574      else if (item is IntValue) return ((IntValue)item).Value;
575      else throw new NotSupportedException("Could not convert any item type to double");
576    }
577    #endregion
578
579    #region events
580    private void RegisterEvents() {
581      Folds.ValueChanged += new EventHandler(Folds_ValueChanged);
582      RegisterClonedAlgorithmsEvents();
583    }
584    private void Folds_ValueChanged(object sender, EventArgs e) {
585      if (ExecutionState != ExecutionState.Prepared)
586        throw new InvalidOperationException("Can not change number of folds if the execution state is not prepared.");
587    }
588
589
590    #region template algorithms events
591    public event EventHandler AlgorithmChanged;
592    private void OnAlgorithmChanged() {
593      EventHandler handler = AlgorithmChanged;
594      if (handler != null) handler(this, EventArgs.Empty);
595      OnProblemChanged();
596      if (Problem == null) ExecutionState = ExecutionState.Stopped;
597    }
598    private void RegisterAlgorithmEvents() {
599      algorithm.ProblemChanged += new EventHandler(Algorithm_ProblemChanged);
600      algorithm.ExecutionStateChanged += new EventHandler(Algorithm_ExecutionStateChanged);
601      if (Problem != null) {
602        Problem.Reset += new EventHandler(Problem_Reset);
603      }
604    }
605    private void DeregisterAlgorithmEvents() {
606      algorithm.ProblemChanged -= new EventHandler(Algorithm_ProblemChanged);
607      algorithm.ExecutionStateChanged -= new EventHandler(Algorithm_ExecutionStateChanged);
608      if (Problem != null) {
609        Problem.Reset -= new EventHandler(Problem_Reset);
610      }
611    }
612    private void Algorithm_ProblemChanged(object sender, EventArgs e) {
613      if (algorithm.Problem != null && !(algorithm.Problem is IDataAnalysisProblem)) {
614        algorithm.Problem = problem;
615        throw new ArgumentException("A cross validation algorithm can only contain DataAnalysisProblems.");
616      }
617      if (problem != null) problem.Reset -= new EventHandler(Problem_Reset);
618      problem = (IDataAnalysisProblem)algorithm.Problem;
619      if (problem != null) problem.Reset += new EventHandler(Problem_Reset);
620      OnProblemChanged();
621    }
622    public event EventHandler ProblemChanged;
623    private void OnProblemChanged() {
624      EventHandler handler = ProblemChanged;
625      if (handler != null) handler(this, EventArgs.Empty);
626      ConfigureProblem();
627    }
628    private void Problem_Reset(object sender, EventArgs e) {
629      ConfigureProblem();
630    }
631    private void ConfigureProblem() {
632      SamplesStart.Value = 0;
633      if (Problem != null) {
634        SamplesEnd.Value = Problem.ProblemData.Dataset.Rows;
635
636        DataAnalysisProblemData problemData = Problem.ProblemData as DataAnalysisProblemData;
637        if (problemData != null) {
638          problemData.TrainingPartitionParameter.Hidden = true;
639          problemData.TestPartitionParameter.Hidden = true;
640        }
641        ISymbolicDataAnalysisProblem symbolicProblem = Problem as ISymbolicDataAnalysisProblem;
642        if (symbolicProblem != null) {
643          symbolicProblem.FitnessCalculationPartitionParameter.Hidden = true;
644          symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
645          symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
646          symbolicProblem.ValidationPartitionParameter.Hidden = true;
647          symbolicProblem.ValidationPartition.Start = 0;
648          symbolicProblem.ValidationPartition.End = 0;
649        }
650      } else
651        SamplesEnd.Value = 0;
652    }
653
654    private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
655      switch (Algorithm.ExecutionState) {
656        case ExecutionState.Prepared:
657          OnPrepared();
658          break;
659        case ExecutionState.Started: throw new InvalidOperationException("Algorithm template can not be started.");
660        case ExecutionState.Paused: throw new InvalidOperationException("Algorithm template can not be paused.");
661        case ExecutionState.Stopped:
662          OnStopped();
663          break;
664      }
665    }
666    #endregion
667
668    #region clonedAlgorithms events
669    private void RegisterClonedAlgorithmsEvents() {
670      clonedAlgorithms.ItemsAdded += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
671      clonedAlgorithms.ItemsRemoved += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
672      clonedAlgorithms.CollectionReset += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
673      foreach (IAlgorithm algorithm in clonedAlgorithms)
674        RegisterClonedAlgorithmEvents(algorithm);
675    }
676    private void DeregisterClonedAlgorithmsEvents() {
677      clonedAlgorithms.ItemsAdded -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
678      clonedAlgorithms.ItemsRemoved -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
679      clonedAlgorithms.CollectionReset -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
680      foreach (IAlgorithm algorithm in clonedAlgorithms)
681        DeregisterClonedAlgorithmEvents(algorithm);
682    }
683    private void ClonedAlgorithms_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
684      foreach (IAlgorithm algorithm in e.Items)
685        RegisterClonedAlgorithmEvents(algorithm);
686    }
687    private void ClonedAlgorithms_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
688      foreach (IAlgorithm algorithm in e.Items)
689        DeregisterClonedAlgorithmEvents(algorithm);
690    }
691    private void ClonedAlgorithms_CollectionReset(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
692      foreach (IAlgorithm algorithm in e.OldItems)
693        DeregisterClonedAlgorithmEvents(algorithm);
694      foreach (IAlgorithm algorithm in e.Items)
695        RegisterClonedAlgorithmEvents(algorithm);
696    }
697    private void RegisterClonedAlgorithmEvents(IAlgorithm algorithm) {
698      algorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
699      algorithm.ExecutionTimeChanged += new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
700      algorithm.Started += new EventHandler(ClonedAlgorithm_Started);
701      algorithm.Paused += new EventHandler(ClonedAlgorithm_Paused);
702      algorithm.Stopped += new EventHandler(ClonedAlgorithm_Stopped);
703    }
704    private void DeregisterClonedAlgorithmEvents(IAlgorithm algorithm) {
705      algorithm.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
706      algorithm.ExecutionTimeChanged -= new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
707      algorithm.Started -= new EventHandler(ClonedAlgorithm_Started);
708      algorithm.Paused -= new EventHandler(ClonedAlgorithm_Paused);
709      algorithm.Stopped -= new EventHandler(ClonedAlgorithm_Stopped);
710    }
711    private void ClonedAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
712      Pause();
713      OnExceptionOccurred(e.Value);
714    }
715    private void ClonedAlgorithm_ExecutionTimeChanged(object sender, EventArgs e) {
716      OnExecutionTimeChanged();
717    }
718
719    private readonly object locker = new object();
720    private readonly object resultLocker = new object();
721    private void ClonedAlgorithm_Started(object sender, EventArgs e) {
722      IAlgorithm algorithm = sender as IAlgorithm;
723      lock (resultLocker) {
724        if (algorithm != null && !results.ContainsKey(algorithm.Name))
725          results.Add(new Result(algorithm.Name, "Contains results for the specific fold.", algorithm.Results));
726      }
727    }
728
729    private void ClonedAlgorithm_Paused(object sender, EventArgs e) {
730      lock (locker) {
731        availableWorkers.Release();
732        if (clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started)) {
733          OnPaused();
734          allAlgorithmsFinished.Set();
735        }
736      }
737    }
738
739    private void ClonedAlgorithm_Stopped(object sender, EventArgs e) {
740      lock (locker) {
741        // if the algorithm was in paused state, its worker has already been released
742        if (availableWorkers.CurrentCount < NumberOfWorkers.Value)
743          availableWorkers.Release();
744        if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped)) {
745          OnStopped();
746          allAlgorithmsFinished.Set();
747        } else if (stopPending && clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped)) {
748          OnStopped();
749          allAlgorithmsFinished.Set();
750        }
751      }
752    }
753    #endregion
754    #endregion
755
756    #region event firing
757    public event EventHandler ExecutionStateChanged;
758    private void OnExecutionStateChanged() {
759      EventHandler handler = ExecutionStateChanged;
760      if (handler != null) handler(this, EventArgs.Empty);
761    }
762    public event EventHandler ExecutionTimeChanged;
763    private void OnExecutionTimeChanged() {
764      EventHandler handler = ExecutionTimeChanged;
765      if (handler != null) handler(this, EventArgs.Empty);
766    }
767    public event EventHandler Prepared;
768    private void OnPrepared() {
769      ExecutionState = ExecutionState.Prepared;
770      EventHandler handler = Prepared;
771      if (handler != null) handler(this, EventArgs.Empty);
772      OnExecutionTimeChanged();
773    }
774    public event EventHandler Started;
775    private void OnStarted() {
776      startPending = false;
777      ExecutionState = ExecutionState.Started;
778      EventHandler handler = Started;
779      if (handler != null) handler(this, EventArgs.Empty);
780    }
781    public event EventHandler Paused;
782    private void OnPaused() {
783      pausePending = false;
784      ExecutionState = ExecutionState.Paused;
785      EventHandler handler = Paused;
786      if (handler != null) handler(this, EventArgs.Empty);
787    }
788    public event EventHandler Stopped;
789    private void OnStopped() {
790      stopPending = false;
791      Dictionary<string, IItem> collectedResults = new Dictionary<string, IItem>();
792      AggregateResultValues(collectedResults);
793      results.AddRange(collectedResults.Select(x => new Result(x.Key, x.Value)).Cast<IResult>().ToArray());
794      clonedAlgorithms.Clear();
795      runsCounter++;
796      runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
797      ExecutionState = ExecutionState.Stopped;
798      EventHandler handler = Stopped;
799      if (handler != null) handler(this, EventArgs.Empty);
800    }
801    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
802    private void OnExceptionOccurred(Exception exception) {
803      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
804      if (handler != null) handler(this, new EventArgs<Exception>(exception));
805    }
806    public event EventHandler StoreAlgorithmInEachRunChanged;
807    private void OnStoreAlgorithmInEachRunChanged() {
808      EventHandler handler = StoreAlgorithmInEachRunChanged;
809      if (handler != null) handler(this, EventArgs.Empty);
810    }
811    #endregion
812  }
813}
Note: See TracBrowser for help on using the repository browser.