source: branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs @ 12800

Last change on this file since 12800 was 12800, checked in by gkronber, 6 years ago

#2434 added support for a partition variable to CrossValidation

File size: 34.8 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Drawing;
25using System.Linq;
26using System.Threading;
27using HeuristicLab.Collections;
28using HeuristicLab.Common;
29using HeuristicLab.Core;
30using HeuristicLab.Data;
31using HeuristicLab.Optimization;
32using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
33using HeuristicLab.Problems.DataAnalysis;
34using HeuristicLab.Problems.DataAnalysis.Symbolic;
35
36namespace HeuristicLab.Algorithms.DataAnalysis {
37  [Item("Cross Validation", "Cross-validation wrapper for data analysis algorithms.")]
38  [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 100)]
39  [StorableClass]
40  public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent {
41    public CrossValidation()
42      : base() {
43      name = ItemName;
44      description = ItemDescription;
45
46      executionState = ExecutionState.Stopped;
47      runs = new RunCollection { OptimizerName = name };
48      runsCounter = 0;
49
50      algorithm = null;
51      clonedAlgorithms = new ItemCollection<IAlgorithm>();
52      results = new ResultCollection();
53
54      folds = 2;
55      numberOfWorkers = new IntValue(1);
56      samplesStart = new IntValue(0);
57      samplesEnd = new IntValue(0);
58      storeAlgorithmInEachRun = false;
59
60      RegisterEvents();
61      if (Algorithm != null) RegisterAlgorithmEvents();
62    }
63
64    public string Filename { get; set; }
65
66    #region persistence and cloning
67    [StorableConstructor]
68    private CrossValidation(bool deserializing)
69      : base(deserializing) {
70    }
71    [StorableHook(HookType.AfterDeserialization)]
72    private void AfterDeserialization() {
73      RegisterEvents();
74      if (Algorithm != null) RegisterAlgorithmEvents();
75
76      // BackwardsCompatibility3.4
77      #region Backwards compatible code, remove with 3.6
78      if (persistenceCompatibilityFolds != null) folds = persistenceCompatibilityFolds.Value;
79      #endregion
80    }
81
82    private CrossValidation(CrossValidation original, Cloner cloner)
83      : base(original, cloner) {
84      executionState = original.executionState;
85      storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
86      runs = cloner.Clone(original.runs);
87      runsCounter = original.runsCounter;
88      algorithm = cloner.Clone(original.algorithm);
89      clonedAlgorithms = cloner.Clone(original.clonedAlgorithms);
90      results = cloner.Clone(original.results);
91
92      folds = original.folds;
93      numberOfWorkers = cloner.Clone(original.numberOfWorkers);
94      samplesStart = cloner.Clone(original.samplesStart);
95      samplesEnd = cloner.Clone(original.samplesEnd);
96      RegisterEvents();
97      if (Algorithm != null) RegisterAlgorithmEvents();
98    }
99    public override IDeepCloneable Clone(Cloner cloner) {
100      return new CrossValidation(this, cloner);
101    }
102
103    #endregion
104
105    #region properties
106    [Storable]
107    private IAlgorithm algorithm;
108    public IAlgorithm Algorithm {
109      get { return algorithm; }
110      set {
111        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
112          throw new InvalidOperationException("Changing the algorithm is only allowed if the CrossValidation is stopped or prepared.");
113        if (algorithm != value) {
114          if (value != null && value.Problem != null && !(value.Problem is IDataAnalysisProblem))
115            throw new ArgumentException("Only algorithms with a DataAnalysisProblem could be used for the cross validation.");
116          if (algorithm != null) DeregisterAlgorithmEvents();
117          algorithm = value;
118          Parameters.Clear();
119
120          if (algorithm != null) {
121            algorithm.StoreAlgorithmInEachRun = false;
122            RegisterAlgorithmEvents();
123            algorithm.Prepare(true);
124            Parameters.AddRange(algorithm.Parameters);
125          }
126          OnAlgorithmChanged();
127          Prepare();
128        }
129      }
130    }
131
132
133    [Storable]
134    private IDataAnalysisProblem problem;
135    public IDataAnalysisProblem Problem {
136      get {
137        if (algorithm == null)
138          return null;
139        return (IDataAnalysisProblem)algorithm.Problem;
140      }
141      set {
142        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
143          throw new InvalidOperationException("Changing the problem is only allowed if the CrossValidation is stopped or prepared.");
144        if (algorithm == null) throw new ArgumentNullException("Could not set a problem before an algorithm was set.");
145        algorithm.Problem = value;
146        problem = value;
147      }
148    }
149
150    IProblem IAlgorithm.Problem {
151      get { return Problem; }
152      set {
153        if (value != null && !ProblemType.IsInstanceOfType(value))
154          throw new ArgumentException("Only DataAnalysisProblems could be used for the cross validation.");
155        Problem = (IDataAnalysisProblem)value;
156      }
157    }
158    public Type ProblemType {
159      get { return typeof(IDataAnalysisProblem); }
160    }
161
162    [Storable]
163    private ItemCollection<IAlgorithm> clonedAlgorithms;
164
165    public IEnumerable<IOptimizer> NestedOptimizers {
166      get {
167        if (Algorithm == null) yield break;
168        yield return Algorithm;
169      }
170    }
171
172    [Storable]
173    private ResultCollection results;
174    public ResultCollection Results {
175      get { return results; }
176    }
177
178    // BackwardsCompatibility3.4
179    #region Backwards compatible code, remove with 3.6
180    [Storable(Name = "folds")]
181    private IntValue persistenceCompatibilityFolds;
182    #endregion
183
184    [Storable(Name = "foldsNew")]
185    private int folds;
186    public int Folds {
187      get { return folds; }
188      set {
189        if (value != folds) {
190          folds = value;
191          partitionVariable = NoPartitionVariable; // setting folds updates the partition variable
192          OnFoldsChanged();
193        }
194      }
195    }
196
197    [Storable]
198    // folds are either specified explicitly or by a varaible from the dataset
199    public const string NoPartitionVariable = "<none>";
200    private string partitionVariable = NoPartitionVariable;
201    public string PartitionVariable {
202      get { return partitionVariable; }
203      set {
204        if (value != partitionVariable) {
205          partitionVariable = value;
206          UpdateNumberOfFolds();
207        }
208      }
209    }
210
211    [Storable]
212    private IntValue samplesStart;
213    public IntValue SamplesStart {
214      get { return samplesStart; }
215    }
216    [Storable]
217    private IntValue samplesEnd;
218    public IntValue SamplesEnd {
219      get { return samplesEnd; }
220    }
221    [Storable]
222    private IntValue numberOfWorkers;
223    public IntValue NumberOfWorkers {
224      get { return numberOfWorkers; }
225    }
226
227    [Storable]
228    private bool storeAlgorithmInEachRun;
229    public bool StoreAlgorithmInEachRun {
230      get { return storeAlgorithmInEachRun; }
231      set {
232        if (storeAlgorithmInEachRun != value) {
233          storeAlgorithmInEachRun = value;
234          OnStoreAlgorithmInEachRunChanged();
235        }
236      }
237    }
238
239    [Storable]
240    private int runsCounter;
241    [Storable]
242    private RunCollection runs;
243    public RunCollection Runs {
244      get { return runs; }
245    }
246
247    [Storable]
248    private ExecutionState executionState;
249    public ExecutionState ExecutionState {
250      get { return executionState; }
251      private set {
252        if (executionState != value) {
253          executionState = value;
254          OnExecutionStateChanged();
255          OnItemImageChanged();
256        }
257      }
258    }
259    public static new Image StaticItemImage {
260      get { return HeuristicLab.Common.Resources.VSImageLibrary.Event; }
261    }
262    public override Image ItemImage {
263      get {
264        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
265        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
266        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
267        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
268        else return base.ItemImage;
269      }
270    }
271
272    public TimeSpan ExecutionTime {
273      get {
274        if (ExecutionState != ExecutionState.Prepared)
275          return TimeSpan.FromMilliseconds(clonedAlgorithms.Select(x => x.ExecutionTime.TotalMilliseconds).Sum());
276        return TimeSpan.Zero;
277      }
278    }
279
280    #endregion
281
282    protected override void OnNameChanged() {
283      base.OnNameChanged();
284      Runs.OptimizerName = Name;
285    }
286
287    public void Prepare() {
288      if (ExecutionState == ExecutionState.Started)
289        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
290      results.Clear();
291      clonedAlgorithms.Clear();
292      if (Algorithm != null) {
293        Algorithm.Prepare();
294        if (Algorithm.ExecutionState == ExecutionState.Prepared) OnPrepared();
295      }
296    }
297    public void Prepare(bool clearRuns) {
298      if (clearRuns) runs.Clear();
299      Prepare();
300    }
301
302    public void Start() {
303      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
304        throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));
305
306      if (Algorithm != null) {
307        //create cloned algorithms
308        if (clonedAlgorithms.Count == 0) {
309
310          for (int i = 0; i < folds; i++) {
311            IAlgorithm clonedAlgorithm = (IAlgorithm)algorithm.Clone();
312            clonedAlgorithm.Name = algorithm.Name + " Fold " + i;
313            IDataAnalysisProblem problem = clonedAlgorithm.Problem as IDataAnalysisProblem;
314            ISymbolicDataAnalysisProblem symbolicProblem = problem as ISymbolicDataAnalysisProblem;
315
316            int trainingStart, trainingEnd, testStart, testEnd;
317            // assumes that partitions are subset of subsequent rows
318            GetTrainingAndTestPartitions(i, out trainingStart, out trainingEnd, out testStart, out testEnd);
319
320            problem.ProblemData.TrainingPartition.Start = SamplesStart.Value;
321            problem.ProblemData.TrainingPartition.End = SamplesEnd.Value;
322            problem.ProblemData.TestPartition.Start = testStart;
323            problem.ProblemData.TestPartition.End = testEnd;
324            DataAnalysisProblemData problemData = problem.ProblemData as DataAnalysisProblemData;
325            if (problemData != null) {
326              problemData.TrainingPartitionParameter.Hidden = false;
327              problemData.TestPartitionParameter.Hidden = false;
328            }
329
330            if (symbolicProblem != null) {
331              symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
332              symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
333            }
334            clonedAlgorithm.Prepare();
335            clonedAlgorithms.Add(clonedAlgorithm);
336          }
337        }
338
339        //start prepared or paused cloned algorithms
340        int startedAlgorithms = 0;
341        foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
342          if (startedAlgorithms < numberOfWorkers.Value) {
343            if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared ||
344                clonedAlgorithm.ExecutionState == ExecutionState.Paused) {
345
346              // start and wait until the alg is started
347              using (var signal = new ManualResetEvent(false)) {
348                EventHandler signalSetter = (sender, args) => { signal.Set(); };
349                clonedAlgorithm.Started += signalSetter;
350                clonedAlgorithm.Start();
351                signal.WaitOne();
352                clonedAlgorithm.Started -= signalSetter;
353
354                startedAlgorithms++;
355              }
356            }
357          }
358        }
359        OnStarted();
360      }
361    }
362
363    private void GetTrainingAndTestPartitions(int fold, out int trainingStart, out int trainingEnd, out int testStart, out int testEnd) {
364      if (partitionVariable == NoPartitionVariable) {
365        // uniform split
366        int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / folds;
367        trainingStart = SamplesStart.Value;
368        trainingEnd = SamplesEnd.Value;
369        testStart = (fold * testSamplesCount) + SamplesStart.Value;
370        testEnd = (fold + 1) == folds ? SamplesEnd.Value : (fold + 1) * testSamplesCount + SamplesStart.Value;
371      } else {
372        // group rowIdx by partition
373        var partition = Problem.ProblemData.Dataset.GetReadOnlyDoubleValues(partitionVariable);
374        var g = Enumerable.Range(SamplesStart.Value, SamplesEnd.Value - SamplesStart.Value).GroupBy(r => partition[r]).OrderBy(r => r.Key).ToList();
375        trainingStart = SamplesStart.Value;
376        trainingEnd = SamplesEnd.Value;
377        testStart = g[fold].Min();
378        testEnd = g[fold].Max() + 1;
379      }
380    }
381
382    private bool pausePending;
383    public void Pause() {
384      if (ExecutionState != ExecutionState.Started)
385        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
386      if (!pausePending) {
387        pausePending = true;
388        PauseAllClonedAlgorithms();
389      }
390    }
391    private void PauseAllClonedAlgorithms() {
392      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
393        if (clonedAlgorithm.ExecutionState == ExecutionState.Started)
394          clonedAlgorithm.Pause();
395      }
396    }
397
398    private bool stopPending;
399    public void Stop() {
400      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
401        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".",
402                                                          ExecutionState));
403      if (!stopPending) {
404        stopPending = true;
405        StopAllClonedAlgorithms();
406      }
407    }
408    private void StopAllClonedAlgorithms() {
409      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
410        if (clonedAlgorithm.ExecutionState == ExecutionState.Started ||
411            clonedAlgorithm.ExecutionState == ExecutionState.Paused)
412          clonedAlgorithm.Stop();
413      }
414    }
415
416    #region collect parameters and results
417    public override void CollectParameterValues(IDictionary<string, IItem> values) {
418      values.Add("Algorithm Name", new StringValue(Name));
419      values.Add("Algorithm Type", new StringValue(GetType().GetPrettyName()));
420      values.Add("Folds", new IntValue(folds));
421
422      if (algorithm != null) {
423        values.Add("CrossValidation Algorithm Name", new StringValue(Algorithm.Name));
424        values.Add("CrossValidation Algorithm Type", new StringValue(Algorithm.GetType().GetPrettyName()));
425        base.CollectParameterValues(values);
426      }
427      if (Problem != null) {
428        values.Add("Problem Name", new StringValue(Problem.Name));
429        values.Add("Problem Type", new StringValue(Problem.GetType().GetPrettyName()));
430        Problem.CollectParameterValues(values);
431      }
432    }
433
434    public void CollectResultValues(IDictionary<string, IItem> results) {
435      var clonedResults = (ResultCollection)this.results.Clone();
436      foreach (var result in clonedResults) {
437        results.Add(result.Name, result.Value);
438      }
439    }
440
441    private void AggregateResultValues(IDictionary<string, IItem> results) {
442      IEnumerable<IRun> runs = clonedAlgorithms.Select(alg => alg.Runs.FirstOrDefault()).Where(run => run != null);
443      IEnumerable<KeyValuePair<string, IItem>> resultCollections = runs.Where(x => x != null).SelectMany(x => x.Results).ToList();
444
445      foreach (IResult result in ExtractAndAggregateResults<IntValue>(resultCollections))
446        results.Add(result.Name, result.Value);
447      foreach (IResult result in ExtractAndAggregateResults<DoubleValue>(resultCollections))
448        results.Add(result.Name, result.Value);
449      foreach (IResult result in ExtractAndAggregateResults<PercentValue>(resultCollections))
450        results.Add(result.Name, result.Value);
451      foreach (IResult result in ExtractAndAggregateRegressionSolutions(resultCollections)) {
452        results.Add(result.Name, result.Value);
453      }
454      foreach (IResult result in ExtractAndAggregateClassificationSolutions(resultCollections)) {
455        results.Add(result.Name, result.Value);
456      }
457      results.Add("Execution Time", new TimeSpanValue(this.ExecutionTime));
458      results.Add("CrossValidation Folds", new RunCollection(runs));
459    }
460
461    private IEnumerable<IResult> ExtractAndAggregateRegressionSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
462      Dictionary<string, List<IRegressionSolution>> resultSolutions = new Dictionary<string, List<IRegressionSolution>>();
463      foreach (var result in resultCollections) {
464        var regressionSolution = result.Value as IRegressionSolution;
465        if (regressionSolution != null) {
466          if (resultSolutions.ContainsKey(result.Key)) {
467            resultSolutions[result.Key].Add(regressionSolution);
468          } else {
469            resultSolutions.Add(result.Key, new List<IRegressionSolution>() { regressionSolution });
470          }
471        }
472      }
473      List<IResult> aggregatedResults = new List<IResult>();
474      foreach (KeyValuePair<string, List<IRegressionSolution>> solutions in resultSolutions) {
475        // clone manually to correctly clone references between cloned root objects
476        Cloner cloner = new Cloner();
477        var problemDataClone = (IRegressionProblemData)cloner.Clone(Problem.ProblemData);
478        // set partitions of problem data clone correctly
479        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
480        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
481        // clone models
482        var ensembleSolution = new RegressionEnsembleSolution(problemDataClone);
483        ensembleSolution.AddRegressionSolutions(solutions.Value);
484
485        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
486      }
487      List<IResult> flattenedResults = new List<IResult>();
488      CollectResultsRecursively("", aggregatedResults, flattenedResults);
489      return flattenedResults;
490    }
491
492    private IEnumerable<IResult> ExtractAndAggregateClassificationSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
493      Dictionary<string, List<IClassificationSolution>> resultSolutions = new Dictionary<string, List<IClassificationSolution>>();
494      foreach (var result in resultCollections) {
495        var classificationSolution = result.Value as IClassificationSolution;
496        if (classificationSolution != null) {
497          if (resultSolutions.ContainsKey(result.Key)) {
498            resultSolutions[result.Key].Add(classificationSolution);
499          } else {
500            resultSolutions.Add(result.Key, new List<IClassificationSolution>() { classificationSolution });
501          }
502        }
503      }
504      var aggregatedResults = new List<IResult>();
505      foreach (KeyValuePair<string, List<IClassificationSolution>> solutions in resultSolutions) {
506        // clone manually to correctly clone references between cloned root objects
507        Cloner cloner = new Cloner();
508        var problemDataClone = (IClassificationProblemData)cloner.Clone(Problem.ProblemData);
509        // set partitions of problem data clone correctly
510        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
511        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
512        // clone models
513        var ensembleSolution = new ClassificationEnsembleSolution(problemDataClone);
514        ensembleSolution.AddClassificationSolutions(solutions.Value);
515
516        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
517      }
518      List<IResult> flattenedResults = new List<IResult>();
519      CollectResultsRecursively("", aggregatedResults, flattenedResults);
520      return flattenedResults;
521    }
522
523    private void CollectResultsRecursively(string path, IEnumerable<IResult> results, IList<IResult> flattenedResults) {
524      foreach (IResult result in results) {
525        flattenedResults.Add(new Result(path + result.Name, result.Value));
526        ResultCollection childCollection = result.Value as ResultCollection;
527        if (childCollection != null) {
528          CollectResultsRecursively(path + result.Name + ".", childCollection, flattenedResults);
529        }
530      }
531    }
532
533    private static IEnumerable<IResult> ExtractAndAggregateResults<T>(IEnumerable<KeyValuePair<string, IItem>> results)
534  where T : class, IItem, new() {
535      Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
536      foreach (var resultValue in results.Where(r => r.Value.GetType() == typeof(T))) {
537        if (!resultValues.ContainsKey(resultValue.Key))
538          resultValues[resultValue.Key] = new List<double>();
539        resultValues[resultValue.Key].Add(ConvertToDouble(resultValue.Value));
540      }
541
542      DoubleValue doubleValue;
543      if (typeof(T) == typeof(PercentValue))
544        doubleValue = new PercentValue();
545      else if (typeof(T) == typeof(DoubleValue))
546        doubleValue = new DoubleValue();
547      else if (typeof(T) == typeof(IntValue))
548        doubleValue = new DoubleValue();
549      else
550        throw new NotSupportedException();
551
552      List<IResult> aggregatedResults = new List<IResult>();
553      foreach (KeyValuePair<string, List<double>> resultValue in resultValues) {
554        doubleValue.Value = resultValue.Value.Average();
555        aggregatedResults.Add(new Result(resultValue.Key + " (average)", (IItem)doubleValue.Clone()));
556        doubleValue.Value = resultValue.Value.StandardDeviation();
557        aggregatedResults.Add(new Result(resultValue.Key + " (std.dev.)", (IItem)doubleValue.Clone()));
558      }
559      return aggregatedResults;
560    }
561
562    private static double ConvertToDouble(IItem item) {
563      if (item is DoubleValue) return ((DoubleValue)item).Value;
564      else if (item is IntValue) return ((IntValue)item).Value;
565      else throw new NotSupportedException("Could not convert any item type to double");
566    }
567    #endregion
568
569    #region events
570    private void RegisterEvents() {
571      SamplesStart.ValueChanged += (s, e) => UpdateNumberOfFolds();
572      SamplesEnd.ValueChanged += (s, e) => UpdateNumberOfFolds();
573
574      RegisterClonedAlgorithmsEvents();
575    }
576
577    private void UpdateNumberOfFolds() {
578      if (ExecutionState == ExecutionState.Paused || executionState == ExecutionState.Started)
579        throw new InvalidOperationException("Can not change number of folds if crossvalidation is paused or started.");
580
581      if (partitionVariable != NoPartitionVariable) {
582        // number of folds is the number of distinct values of the partition variable in the range [SamplesStart..SamplesEnd[
583        var ds = Problem.ProblemData.Dataset;
584        var partitionValues = ds.GetDoubleValues(PartitionVariable, Enumerable.Range(SamplesStart.Value, SamplesEnd.Value - SamplesStart.Value)).Distinct().Count();
585        folds = partitionValues;
586        OnFoldsChanged();
587      }
588    }
589
590
591    #region template algorithms events
592    public event EventHandler AlgorithmChanged;
593    private void OnAlgorithmChanged() {
594      EventHandler handler = AlgorithmChanged;
595      if (handler != null) handler(this, EventArgs.Empty);
596      OnProblemChanged();
597      if (Problem == null) ExecutionState = ExecutionState.Stopped;
598    }
599    private void RegisterAlgorithmEvents() {
600      algorithm.ProblemChanged += new EventHandler(Algorithm_ProblemChanged);
601      algorithm.ExecutionStateChanged += new EventHandler(Algorithm_ExecutionStateChanged);
602      if (Problem != null) Problem.Reset += new EventHandler(Problem_Reset);
603    }
604    private void DeregisterAlgorithmEvents() {
605      algorithm.ProblemChanged -= new EventHandler(Algorithm_ProblemChanged);
606      algorithm.ExecutionStateChanged -= new EventHandler(Algorithm_ExecutionStateChanged);
607      if (Problem != null) Problem.Reset -= new EventHandler(Problem_Reset);
608    }
609    private void Algorithm_ProblemChanged(object sender, EventArgs e) {
610      if (algorithm.Problem != null && !(algorithm.Problem is IDataAnalysisProblem)) {
611        algorithm.Problem = problem;
612        throw new ArgumentException("A cross validation algorithm can only contain DataAnalysisProblems.");
613      }
614      if (problem != null) problem.Reset -= new EventHandler(Problem_Reset);
615      problem = (IDataAnalysisProblem)algorithm.Problem;
616      if (problem != null) problem.Reset += new EventHandler(Problem_Reset);
617      OnProblemChanged();
618    }
619    public event EventHandler ProblemChanged;
620    private void OnProblemChanged() {
621      ConfigureProblem();
622      EventHandler handler = ProblemChanged;
623      if (handler != null) handler(this, EventArgs.Empty);
624    }
625
626    private void Problem_Reset(object sender, EventArgs e) {
627      ConfigureProblem();
628    }
629
630    private void ConfigureProblem() {
631      folds = 2;
632      partitionVariable = NoPartitionVariable;
633      SamplesStart.Value = 0;
634      if (Problem != null) {
635        SamplesEnd.Value = Problem.ProblemData.Dataset.Rows;
636
637        DataAnalysisProblemData problemData = Problem.ProblemData as DataAnalysisProblemData;
638        if (problemData != null) {
639          problemData.TrainingPartitionParameter.Hidden = true;
640          problemData.TestPartitionParameter.Hidden = true;
641        }
642        ISymbolicDataAnalysisProblem symbolicProblem = Problem as ISymbolicDataAnalysisProblem;
643        if (symbolicProblem != null) {
644          symbolicProblem.FitnessCalculationPartitionParameter.Hidden = true;
645          symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
646          symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
647          symbolicProblem.ValidationPartitionParameter.Hidden = true;
648          symbolicProblem.ValidationPartition.Start = 0;
649          symbolicProblem.ValidationPartition.End = 0;
650        }
651      } else
652        SamplesEnd.Value = 0;
653    }
654
655    private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
656      switch (Algorithm.ExecutionState) {
657        case ExecutionState.Prepared: OnPrepared();
658          break;
659        case ExecutionState.Started: throw new InvalidOperationException("Algorithm template can not be started.");
660        case ExecutionState.Paused: throw new InvalidOperationException("Algorithm template can not be paused.");
661        case ExecutionState.Stopped: OnStopped();
662          break;
663      }
664    }
665    #endregion
666
667    #region clonedAlgorithms events
668    private void RegisterClonedAlgorithmsEvents() {
669      clonedAlgorithms.ItemsAdded += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
670      clonedAlgorithms.ItemsRemoved += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
671      clonedAlgorithms.CollectionReset += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
672      foreach (IAlgorithm algorithm in clonedAlgorithms)
673        RegisterClonedAlgorithmEvents(algorithm);
674    }
675    private void DeregisterClonedAlgorithmsEvents() {
676      clonedAlgorithms.ItemsAdded -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
677      clonedAlgorithms.ItemsRemoved -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
678      clonedAlgorithms.CollectionReset -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
679      foreach (IAlgorithm algorithm in clonedAlgorithms)
680        DeregisterClonedAlgorithmEvents(algorithm);
681    }
682    private void ClonedAlgorithms_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
683      foreach (IAlgorithm algorithm in e.Items)
684        RegisterClonedAlgorithmEvents(algorithm);
685    }
686    private void ClonedAlgorithms_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
687      foreach (IAlgorithm algorithm in e.Items)
688        DeregisterClonedAlgorithmEvents(algorithm);
689    }
690    private void ClonedAlgorithms_CollectionReset(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
691      foreach (IAlgorithm algorithm in e.OldItems)
692        DeregisterClonedAlgorithmEvents(algorithm);
693      foreach (IAlgorithm algorithm in e.Items)
694        RegisterClonedAlgorithmEvents(algorithm);
695    }
696    private void RegisterClonedAlgorithmEvents(IAlgorithm algorithm) {
697      algorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
698      algorithm.ExecutionTimeChanged += new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
699      algorithm.Started += new EventHandler(ClonedAlgorithm_Started);
700      algorithm.Paused += new EventHandler(ClonedAlgorithm_Paused);
701      algorithm.Stopped += new EventHandler(ClonedAlgorithm_Stopped);
702    }
703    private void DeregisterClonedAlgorithmEvents(IAlgorithm algorithm) {
704      algorithm.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
705      algorithm.ExecutionTimeChanged -= new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
706      algorithm.Started -= new EventHandler(ClonedAlgorithm_Started);
707      algorithm.Paused -= new EventHandler(ClonedAlgorithm_Paused);
708      algorithm.Stopped -= new EventHandler(ClonedAlgorithm_Stopped);
709    }
710    private void ClonedAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
711      OnExceptionOccurred(e.Value);
712    }
713    private void ClonedAlgorithm_ExecutionTimeChanged(object sender, EventArgs e) {
714      OnExecutionTimeChanged();
715    }
716
717    private readonly object locker = new object();
718    private readonly object resultLocker = new object();
719    private void ClonedAlgorithm_Started(object sender, EventArgs e) {
720      IAlgorithm algorithm = sender as IAlgorithm;
721      lock (resultLocker) {
722        if (algorithm != null && !results.ContainsKey(algorithm.Name))
723          results.Add(new Result(algorithm.Name, "Contains results for the specific fold.", algorithm.Results));
724      }
725    }
726
727    private void ClonedAlgorithm_Paused(object sender, EventArgs e) {
728      lock (locker) {
729        if (pausePending && clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started))
730          OnPaused();
731      }
732    }
733
734    private void ClonedAlgorithm_Stopped(object sender, EventArgs e) {
735      lock (locker) {
736        if (!stopPending && ExecutionState == ExecutionState.Started) {
737          IAlgorithm preparedAlgorithm = clonedAlgorithms.FirstOrDefault(alg => alg.ExecutionState == ExecutionState.Prepared ||
738                                                                                alg.ExecutionState == ExecutionState.Paused);
739          if (preparedAlgorithm != null) preparedAlgorithm.Start();
740        }
741        if (ExecutionState != ExecutionState.Stopped) {
742          if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped))
743            OnStopped();
744          else if (stopPending &&
745                   clonedAlgorithms.All(
746                     alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped))
747            OnStopped();
748        }
749      }
750    }
751    #endregion
752    #endregion
753
754    #region event firing
755
756    public event EventHandler FoldsChanged;
757    private void OnFoldsChanged() {
758      EventHandler handler = FoldsChanged;
759      if (handler != null) handler(this, EventArgs.Empty);
760    }
761    public event EventHandler ExecutionStateChanged;
762    private void OnExecutionStateChanged() {
763      EventHandler handler = ExecutionStateChanged;
764      if (handler != null) handler(this, EventArgs.Empty);
765    }
766    public event EventHandler ExecutionTimeChanged;
767    private void OnExecutionTimeChanged() {
768      EventHandler handler = ExecutionTimeChanged;
769      if (handler != null) handler(this, EventArgs.Empty);
770    }
771    public event EventHandler Prepared;
772    private void OnPrepared() {
773      ExecutionState = ExecutionState.Prepared;
774      EventHandler handler = Prepared;
775      if (handler != null) handler(this, EventArgs.Empty);
776      OnExecutionTimeChanged();
777    }
778    public event EventHandler Started;
779    private void OnStarted() {
780      ExecutionState = ExecutionState.Started;
781      EventHandler handler = Started;
782      if (handler != null) handler(this, EventArgs.Empty);
783    }
784    public event EventHandler Paused;
785    private void OnPaused() {
786      pausePending = false;
787      ExecutionState = ExecutionState.Paused;
788      EventHandler handler = Paused;
789      if (handler != null) handler(this, EventArgs.Empty);
790    }
791    public event EventHandler Stopped;
792    private void OnStopped() {
793      stopPending = false;
794      Dictionary<string, IItem> collectedResults = new Dictionary<string, IItem>();
795      AggregateResultValues(collectedResults);
796      results.AddRange(collectedResults.Select(x => new Result(x.Key, x.Value)).Cast<IResult>().ToArray());
797      runsCounter++;
798      runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
799      ExecutionState = ExecutionState.Stopped;
800      EventHandler handler = Stopped;
801      if (handler != null) handler(this, EventArgs.Empty);
802    }
803    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
804    private void OnExceptionOccurred(Exception exception) {
805      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
806      if (handler != null) handler(this, new EventArgs<Exception>(exception));
807    }
808    public event EventHandler StoreAlgorithmInEachRunChanged;
809    private void OnStoreAlgorithmInEachRunChanged() {
810      EventHandler handler = StoreAlgorithmInEachRunChanged;
811      if (handler != null) handler(this, EventArgs.Empty);
812    }
813    #endregion
814  }
815}
Note: See TracBrowser for help on using the repository browser.