Free cookie consent management tool by TermsFeed Policy Generator

source: branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs

Last change on this file was 17835, checked in by bburlacu, 4 years ago

#3102: Add ClassificationProblemData constructor that explicitly takes class names and positive class value arguments, adapt code.

File size: 36.3 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Drawing;
25using System.Linq;
26using System.Threading;
27using System.Threading.Tasks;
28using HeuristicLab.Collections;
29using HeuristicLab.Common;
30using HeuristicLab.Core;
31using HeuristicLab.Data;
32using HeuristicLab.Optimization;
33using HEAL.Attic;
34using HeuristicLab.Problems.DataAnalysis;
35using HeuristicLab.Problems.DataAnalysis.Symbolic;
36using HeuristicLab.Random;
37using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
38using HeuristicLab.Problems.DataAnalysis.Symbolic.Classification;
39
40namespace HeuristicLab.Algorithms.DataAnalysis {
41  [Item("Cross Validation (CV)", "Cross-validation wrapper for data analysis algorithms.")]
42  [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 100)]
43  [StorableType("1C622121-AE5B-42FD-831C-FCA8F8E0AF8D")]
44  public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent {
45    [Storable]
46    private int seed;
47
48    private SemaphoreSlim availableWorkers; // limits the number of concurrent algorithm executions
49    private ManualResetEventSlim allAlgorithmsFinished; // this indicates that all started algorithms have been paused or stopped
50
51    public CrossValidation()
52      : base() {
53      name = ItemName;
54      description = ItemDescription;
55
56      executionState = ExecutionState.Stopped;
57      runs = new RunCollection { OptimizerName = name };
58      runsCounter = 0;
59
60      algorithm = null;
61      clonedAlgorithms = new ItemCollection<IAlgorithm>();
62      results = new ResultCollection();
63
64      folds = new IntValue(2);
65      numberOfWorkers = new IntValue(1);
66      samplesStart = new IntValue(0);
67      samplesEnd = new IntValue(0);
68      shuffleSamples = new BoolValue(false);
69      storeAlgorithmInEachRun = false;
70
71      RegisterEvents();
72      if (Algorithm != null) RegisterAlgorithmEvents();
73    }
74
75    public string Filename { get; set; }
76
77    #region persistence and cloning
78    [StorableConstructor]
79    private CrossValidation(StorableConstructorFlag _) : base(_) {
80    }
81    [StorableHook(HookType.AfterDeserialization)]
82    private void AfterDeserialization() {
83      // BackwardsCompatibility3.3
84      #region Backwards compatible code, remove with 3.4
85      if (shuffleSamples == null) shuffleSamples = new BoolValue(false);
86      #endregion
87
88      RegisterEvents();
89      if (Algorithm != null) RegisterAlgorithmEvents();
90    }
91
92    private CrossValidation(CrossValidation original, Cloner cloner)
93      : base(original, cloner) {
94      executionState = original.executionState;
95      storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
96      runs = cloner.Clone(original.runs);
97      runsCounter = original.runsCounter;
98      algorithm = cloner.Clone(original.algorithm);
99      clonedAlgorithms = cloner.Clone(original.clonedAlgorithms);
100      results = cloner.Clone(original.results);
101
102      folds = cloner.Clone(original.folds);
103      numberOfWorkers = cloner.Clone(original.numberOfWorkers);
104      samplesStart = cloner.Clone(original.samplesStart);
105      samplesEnd = cloner.Clone(original.samplesEnd);
106      shuffleSamples = cloner.Clone(original.shuffleSamples);
107      seed = original.seed;
108
109      RegisterEvents();
110      if (Algorithm != null) RegisterAlgorithmEvents();
111    }
112    public override IDeepCloneable Clone(Cloner cloner) {
113      return new CrossValidation(this, cloner);
114    }
115
116    #endregion
117
118    #region properties
119    [Storable]
120    private IAlgorithm algorithm;
121    public IAlgorithm Algorithm {
122      get { return algorithm; }
123      set {
124        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
125          throw new InvalidOperationException("Changing the algorithm is only allowed if the CrossValidation is stopped or prepared.");
126        if (algorithm != value) {
127          if (value != null && value.Problem != null && !(value.Problem is IDataAnalysisProblem))
128            throw new ArgumentException("Only algorithms with a DataAnalysisProblem could be used for the cross validation.");
129          if (algorithm != null) DeregisterAlgorithmEvents();
130          algorithm = value;
131          Parameters.Clear();
132
133          if (algorithm != null) {
134            algorithm.StoreAlgorithmInEachRun = false;
135            RegisterAlgorithmEvents();
136            algorithm.Prepare(true);
137            Parameters.AddRange(algorithm.Parameters);
138          }
139          OnAlgorithmChanged();
140          Prepare();
141        }
142      }
143    }
144
145
146    [Storable]
147    private IDataAnalysisProblem problem;
148    public IDataAnalysisProblem Problem {
149      get {
150        if (algorithm == null)
151          return null;
152        return (IDataAnalysisProblem)algorithm.Problem;
153      }
154      set {
155        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
156          throw new InvalidOperationException("Changing the problem is only allowed if the CrossValidation is stopped or prepared.");
157        if (algorithm == null) throw new ArgumentNullException("Could not set a problem before an algorithm was set.");
158        algorithm.Problem = value;
159        problem = value;
160      }
161    }
162
163    IProblem IAlgorithm.Problem {
164      get { return Problem; }
165      set {
166        if (value != null && !ProblemType.IsInstanceOfType(value))
167          throw new ArgumentException("Only DataAnalysisProblems could be used for the cross validation.");
168        Problem = (IDataAnalysisProblem)value;
169      }
170    }
171    public Type ProblemType {
172      get { return typeof(IDataAnalysisProblem); }
173    }
174
175    [Storable]
176    private ItemCollection<IAlgorithm> clonedAlgorithms;
177
178    public IEnumerable<IOptimizer> NestedOptimizers {
179      get {
180        if (Algorithm == null) yield break;
181        yield return Algorithm;
182      }
183    }
184
185    [Storable]
186    private ResultCollection results;
187    public ResultCollection Results {
188      get { return results; }
189    }
190    [Storable]
191    private BoolValue shuffleSamples;
192    public BoolValue ShuffleSamples {
193      get { return shuffleSamples; }
194    }
195    [Storable]
196    private IntValue folds;
197    public IntValue Folds {
198      get { return folds; }
199    }
200    [Storable]
201    private IntValue samplesStart;
202    public IntValue SamplesStart {
203      get { return samplesStart; }
204    }
205    [Storable]
206    private IntValue samplesEnd;
207    public IntValue SamplesEnd {
208      get { return samplesEnd; }
209    }
210    [Storable]
211    private IntValue numberOfWorkers;
212    public IntValue NumberOfWorkers {
213      get { return numberOfWorkers; }
214    }
215
216    [Storable]
217    private bool storeAlgorithmInEachRun;
218    public bool StoreAlgorithmInEachRun {
219      get { return storeAlgorithmInEachRun; }
220      set {
221        if (storeAlgorithmInEachRun != value) {
222          storeAlgorithmInEachRun = value;
223          OnStoreAlgorithmInEachRunChanged();
224        }
225      }
226    }
227
228    [Storable]
229    private int runsCounter;
230    [Storable]
231    private RunCollection runs;
232    public RunCollection Runs {
233      get { return runs; }
234    }
235
236    [Storable]
237    private ExecutionState executionState;
238    public ExecutionState ExecutionState {
239      get { return executionState; }
240      private set {
241        if (executionState != value) {
242          executionState = value;
243          OnExecutionStateChanged();
244          OnItemImageChanged();
245        }
246      }
247    }
248    public static new Image StaticItemImage {
249      get { return HeuristicLab.Common.Resources.VSImageLibrary.Event; }
250    }
251    public override Image ItemImage {
252      get {
253        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
254        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
255        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
256        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
257        else return base.ItemImage;
258      }
259    }
260
261    public TimeSpan ExecutionTime {
262      get {
263        if (ExecutionState != ExecutionState.Prepared)
264          return TimeSpan.FromMilliseconds(clonedAlgorithms.Select(x => x.ExecutionTime.TotalMilliseconds).Sum());
265        return TimeSpan.Zero;
266      }
267    }
268    #endregion
269
270    protected override void OnNameChanged() {
271      base.OnNameChanged();
272      Runs.OptimizerName = Name;
273    }
274
275    public void Prepare() {
276      if (startPending) return;
277      if (ExecutionState == ExecutionState.Started)
278        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
279      results.Clear();
280      clonedAlgorithms.Clear();
281      if (Algorithm != null) {
282        Algorithm.Prepare();
283        if (Algorithm.ExecutionState == ExecutionState.Prepared) OnPrepared();
284      }
285    }
286    public void Prepare(bool clearRuns) {
287      if (clearRuns) runs.Clear();
288      Prepare();
289    }
290
291    private bool startPending;
292    public void Start() {
293      Start(CancellationToken.None);
294    }
295    public void Start(CancellationToken cancellationToken) {
296      lock (locker) {
297        if (startPending) return;
298        startPending = true;
299      }
300
301      try {
302        if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
303          throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));
304        seed = RandomSeedGenerator.GetSeed();
305
306        if (Algorithm == null) return;
307        //create cloned algorithms
308        if (clonedAlgorithms.Count == 0) {
309          int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / Folds.Value;
310          IDataset shuffledDataset = null;
311          for (int i = 0; i < Folds.Value; i++) {
312            var cloner = new Cloner();
313            if (ShuffleSamples.Value) {
314              var random = new FastRandom(seed);
315              var dataAnalysisProblem = (IDataAnalysisProblem)algorithm.Problem;
316              var dataset = (Dataset)dataAnalysisProblem.ProblemData.Dataset;
317              shuffledDataset = shuffledDataset ?? dataset.Shuffle(random);
318              cloner.RegisterClonedObject(dataset, shuffledDataset);
319            }
320            IAlgorithm clonedAlgorithm = cloner.Clone(Algorithm);
321            clonedAlgorithm.Name = algorithm.Name + " Fold " + i;
322            IDataAnalysisProblem problem = clonedAlgorithm.Problem as IDataAnalysisProblem;
323            ISymbolicDataAnalysisProblem symbolicProblem = problem as ISymbolicDataAnalysisProblem;
324
325            int testStart = (i * testSamplesCount) + SamplesStart.Value;
326            int testEnd = (i + 1) == Folds.Value ? SamplesEnd.Value : (i + 1) * testSamplesCount + SamplesStart.Value;
327
328            problem.ProblemData.TrainingPartition.Start = SamplesStart.Value;
329            problem.ProblemData.TrainingPartition.End = SamplesEnd.Value;
330            problem.ProblemData.TestPartition.Start = testStart;
331            problem.ProblemData.TestPartition.End = testEnd;
332            DataAnalysisProblemData problemData = problem.ProblemData as DataAnalysisProblemData;
333            if (problemData != null) {
334              problemData.TrainingPartitionParameter.Hidden = false;
335              problemData.TestPartitionParameter.Hidden = false;
336            }
337
338            if (symbolicProblem != null) {
339              symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
340              symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
341            }
342
343            // We need to set the estimation limits because they are recalculated by the problem
344            // whenever the data partitions change.
345            // Instead of explicitly handling all types we could also check the parameters-collection
346            // for a parameter with name "EstimationLimits".
347            SetEstimationLimits(problem, new[] { typeof(SymbolicRegressionSingleObjectiveProblem),
348                                                 typeof(SymbolicRegressionMultiObjectiveProblem),
349                                                 typeof(SymbolicClassificationSingleObjectiveProblem),
350                                                 typeof(SymbolicClassificationMultiObjectiveProblem) });
351
352            clonedAlgorithm.Prepare();
353            clonedAlgorithms.Add(clonedAlgorithm);
354          }
355        }
356
357        OnStarted();
358      } finally {
359        if (startPending) startPending = false;
360      }
361
362      availableWorkers = new SemaphoreSlim(NumberOfWorkers.Value, NumberOfWorkers.Value);
363      allAlgorithmsFinished = new ManualResetEventSlim(false);
364
365      var startedTasks = new List<Task>(clonedAlgorithms.Count);
366
367      //start prepared or paused cloned algorithms
368      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
369        if (pausePending || stopPending || ExecutionState != ExecutionState.Started) break;
370        if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared ||
371            clonedAlgorithm.ExecutionState == ExecutionState.Paused) {
372          availableWorkers.Wait();
373          lock (locker) {
374            if (pausePending || stopPending || ExecutionState != ExecutionState.Started) break;
375            var task = clonedAlgorithm.StartAsync(cancellationToken);
376            startedTasks.Add(task);
377          }
378        }
379      }
380
381      allAlgorithmsFinished.Wait();
382
383      Task.WaitAll(startedTasks.ToArray()); // to get exceptions not handled within the tasks
384    }
385
386    public async Task StartAsync() { await StartAsync(CancellationToken.None); }
387    public async Task StartAsync(CancellationToken cancellationToken) {
388      await AsyncHelper.DoAsync(Start, cancellationToken);
389    }
390
391    private bool pausePending;
392    public void Pause() {
393      if (startPending) return;
394      if (ExecutionState != ExecutionState.Started)
395        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
396      if (!pausePending) {
397        pausePending = true;
398        lock (locker) {
399          var toPause = clonedAlgorithms.Where(x => x.ExecutionState == ExecutionState.Started).ToList();
400          foreach (var optimizer in toPause) {
401            // a race-condition may occur when the optimizer has changed the state by itself in the meantime
402            try { optimizer.Pause(); } catch (InvalidOperationException) { }
403          }
404        }
405      }
406    }
407
408    private bool stopPending;
409    public void Stop() {
410      if (startPending) return;
411      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
412        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".",
413                                                          ExecutionState));
414      if (!stopPending) {
415        stopPending = true;
416        lock (locker) {
417          var toStop = clonedAlgorithms.Where(x => x.ExecutionState == ExecutionState.Started || x.ExecutionState == ExecutionState.Paused).ToList();
418          foreach (var optimizer in toStop) {
419            // a race-condition may occur when the optimizer has changed the state by itself in the meantime
420            try { optimizer.Stop(); } catch (InvalidOperationException) { }
421          }
422        }
423      }
424    }
425
426    #region collect parameters and results
427    public override void CollectParameterValues(IDictionary<string, IItem> values) {
428      values.Add("Algorithm Name", new StringValue(Name));
429      values.Add("Algorithm Type", new StringValue(GetType().GetPrettyName()));
430      values.Add("Folds", new IntValue(Folds.Value));
431
432      if (algorithm != null) {
433        values.Add("CrossValidation Algorithm Name", new StringValue(Algorithm.Name));
434        values.Add("CrossValidation Algorithm Type", new StringValue(Algorithm.GetType().GetPrettyName()));
435        base.CollectParameterValues(values);
436      }
437      if (Problem != null) {
438        values.Add("Problem Name", new StringValue(Problem.Name));
439        values.Add("Problem Type", new StringValue(Problem.GetType().GetPrettyName()));
440        Problem.CollectParameterValues(values);
441      }
442    }
443
444    public void CollectResultValues(IDictionary<string, IItem> results) {
445      var clonedResults = (ResultCollection)this.results.Clone();
446      foreach (var result in clonedResults) {
447        results.Add(result.Name, result.Value);
448      }
449    }
450
451    private void AggregateResultValues(IDictionary<string, IItem> results) {
452      IEnumerable<IRun> runs = clonedAlgorithms.Select(alg => alg.Runs.FirstOrDefault()).Where(run => run != null);
453      IEnumerable<KeyValuePair<string, IItem>> resultCollections = runs.Where(x => x != null).SelectMany(x => x.Results).ToList();
454
455      foreach (IResult result in ExtractAndAggregateResults<IntValue>(resultCollections))
456        results.Add(result.Name, result.Value);
457      foreach (IResult result in ExtractAndAggregateResults<DoubleValue>(resultCollections))
458        results.Add(result.Name, result.Value);
459      foreach (IResult result in ExtractAndAggregateResults<PercentValue>(resultCollections))
460        results.Add(result.Name, result.Value);
461      foreach (IResult result in ExtractAndAggregateRegressionSolutions(resultCollections)) {
462        results.Add(result.Name, result.Value);
463      }
464      foreach (IResult result in ExtractAndAggregateClassificationSolutions(resultCollections)) {
465        results.Add(result.Name, result.Value);
466      }
467      results.Add("Execution Time", new TimeSpanValue(this.ExecutionTime));
468      results.Add("CrossValidation Folds", new RunCollection(runs));
469    }
470
471    private IEnumerable<IResult> ExtractAndAggregateRegressionSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
472      Dictionary<string, List<IRegressionSolution>> resultSolutions = new Dictionary<string, List<IRegressionSolution>>();
473      foreach (var result in resultCollections) {
474        var regressionSolution = result.Value as IRegressionSolution;
475        if (regressionSolution != null) {
476          if (resultSolutions.ContainsKey(result.Key)) {
477            resultSolutions[result.Key].Add(regressionSolution);
478          } else {
479            resultSolutions.Add(result.Key, new List<IRegressionSolution>() { regressionSolution });
480          }
481        }
482      }
483      List<IResult> aggregatedResults = new List<IResult>();
484      foreach (KeyValuePair<string, List<IRegressionSolution>> solutions in resultSolutions) {
485        // clone manually to correctly clone references between cloned root objects
486        Cloner cloner = new Cloner();
487        if (ShuffleSamples.Value) {
488          var dataset = (Dataset)Problem.ProblemData.Dataset;
489          var random = new FastRandom(seed);
490          var shuffledDataset = dataset.Shuffle(random);
491          cloner.RegisterClonedObject(dataset, shuffledDataset);
492        }
493        var problemDataClone = (IRegressionProblemData)cloner.Clone(Problem.ProblemData);
494        // set partitions of problem data clone correctly
495        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
496        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
497        // clone models
498        var ensembleSolution = new RegressionEnsembleSolution(problemDataClone);
499        ensembleSolution.AddRegressionSolutions(solutions.Value);
500
501        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
502      }
503      List<IResult> flattenedResults = new List<IResult>();
504      CollectResultsRecursively("", aggregatedResults, flattenedResults);
505      return flattenedResults;
506    }
507
508    private IEnumerable<IResult> ExtractAndAggregateClassificationSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
509      Dictionary<string, List<IClassificationSolution>> resultSolutions = new Dictionary<string, List<IClassificationSolution>>();
510      foreach (var result in resultCollections) {
511        var classificationSolution = result.Value as IClassificationSolution;
512        if (classificationSolution != null) {
513          if (resultSolutions.ContainsKey(result.Key)) {
514            resultSolutions[result.Key].Add(classificationSolution);
515          } else {
516            resultSolutions.Add(result.Key, new List<IClassificationSolution>() { classificationSolution });
517          }
518        }
519      }
520      var aggregatedResults = new List<IResult>();
521      foreach (KeyValuePair<string, List<IClassificationSolution>> solutions in resultSolutions) {
522        // at least one algorithm (GBT with logistic regression loss) produces a classification solution even though the original problem is a regression problem.
523        var dataset = (Dataset)Problem.ProblemData.Dataset;
524        if (ShuffleSamples.Value) {
525          var random = new FastRandom(seed);
526          dataset = dataset.Shuffle(random);
527        }
528        var problemData = (IClassificationProblemData)Problem.ProblemData;
529        var problemDataClone = new ClassificationProblemData(dataset, problemData.AllowedInputVariables, problemData.TargetVariable, problemData.ClassNames, problemData.PositiveClass);
530        // set partitions of problem data clone correctly
531        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
532        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
533        // clone models
534        var ensembleSolution = new ClassificationEnsembleSolution(problemDataClone);
535        ensembleSolution.AddClassificationSolutions(solutions.Value);
536
537        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
538      }
539      List<IResult> flattenedResults = new List<IResult>();
540      CollectResultsRecursively("", aggregatedResults, flattenedResults);
541      return flattenedResults;
542    }
543
544    private void CollectResultsRecursively(string path, IEnumerable<IResult> results, IList<IResult> flattenedResults) {
545      foreach (IResult result in results) {
546        flattenedResults.Add(new Result(path + result.Name, result.Value));
547        ResultCollection childCollection = result.Value as ResultCollection;
548        if (childCollection != null) {
549          CollectResultsRecursively(path + result.Name + ".", childCollection, flattenedResults);
550        }
551      }
552    }
553
554    private static IEnumerable<IResult> ExtractAndAggregateResults<T>(IEnumerable<KeyValuePair<string, IItem>> results)
555  where T : class, IItem, new() {
556      Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
557      foreach (var resultValue in results.Where(r => r.Value.GetType() == typeof(T))) {
558        if (!resultValues.ContainsKey(resultValue.Key))
559          resultValues[resultValue.Key] = new List<double>();
560        resultValues[resultValue.Key].Add(ConvertToDouble(resultValue.Value));
561      }
562
563      DoubleValue doubleValue;
564      if (typeof(T) == typeof(PercentValue))
565        doubleValue = new PercentValue();
566      else if (typeof(T) == typeof(DoubleValue))
567        doubleValue = new DoubleValue();
568      else if (typeof(T) == typeof(IntValue))
569        doubleValue = new DoubleValue();
570      else
571        throw new NotSupportedException();
572
573      List<IResult> aggregatedResults = new List<IResult>();
574      foreach (KeyValuePair<string, List<double>> resultValue in resultValues) {
575        doubleValue.Value = resultValue.Value.Average();
576        aggregatedResults.Add(new Result(resultValue.Key + " (average)", (IItem)doubleValue.Clone()));
577        doubleValue.Value = resultValue.Value.StandardDeviation();
578        aggregatedResults.Add(new Result(resultValue.Key + " (std.dev.)", (IItem)doubleValue.Clone()));
579      }
580      return aggregatedResults;
581    }
582
583    private static double ConvertToDouble(IItem item) {
584      if (item is DoubleValue) return ((DoubleValue)item).Value;
585      else if (item is IntValue) return ((IntValue)item).Value;
586      else throw new NotSupportedException("Could not convert any item type to double");
587    }
588    #endregion
589
590    #region events
591    private void RegisterEvents() {
592      Folds.ValueChanged += new EventHandler(Folds_ValueChanged);
593      RegisterClonedAlgorithmsEvents();
594    }
595    private void Folds_ValueChanged(object sender, EventArgs e) {
596      if (ExecutionState != ExecutionState.Prepared)
597        throw new InvalidOperationException("Can not change number of folds if the execution state is not prepared.");
598    }
599
600
601    #region template algorithms events
602    public event EventHandler AlgorithmChanged;
603    private void OnAlgorithmChanged() {
604      EventHandler handler = AlgorithmChanged;
605      if (handler != null) handler(this, EventArgs.Empty);
606      OnProblemChanged();
607      if (Problem == null) ExecutionState = ExecutionState.Stopped;
608    }
609    private void RegisterAlgorithmEvents() {
610      algorithm.ProblemChanged += new EventHandler(Algorithm_ProblemChanged);
611      algorithm.ExecutionStateChanged += new EventHandler(Algorithm_ExecutionStateChanged);
612      if (Problem != null) {
613        Problem.Reset += new EventHandler(Problem_Reset);
614      }
615    }
616    private void DeregisterAlgorithmEvents() {
617      algorithm.ProblemChanged -= new EventHandler(Algorithm_ProblemChanged);
618      algorithm.ExecutionStateChanged -= new EventHandler(Algorithm_ExecutionStateChanged);
619      if (Problem != null) {
620        Problem.Reset -= new EventHandler(Problem_Reset);
621      }
622    }
623    private void Algorithm_ProblemChanged(object sender, EventArgs e) {
624      if (algorithm.Problem != null && !(algorithm.Problem is IDataAnalysisProblem)) {
625        algorithm.Problem = problem;
626        throw new ArgumentException("A cross validation algorithm can only contain DataAnalysisProblems.");
627      }
628      if (problem != null) problem.Reset -= new EventHandler(Problem_Reset);
629      problem = (IDataAnalysisProblem)algorithm.Problem;
630      if (problem != null) problem.Reset += new EventHandler(Problem_Reset);
631      OnProblemChanged();
632    }
633    public event EventHandler ProblemChanged;
634    private void OnProblemChanged() {
635      EventHandler handler = ProblemChanged;
636      if (handler != null) handler(this, EventArgs.Empty);
637      ConfigureProblem();
638    }
639    private void Problem_Reset(object sender, EventArgs e) {
640      ConfigureProblem();
641    }
642    private void ConfigureProblem() {
643      SamplesStart.Value = 0;
644      if (Problem != null) {
645        SamplesEnd.Value = Problem.ProblemData.Dataset.Rows;
646
647        DataAnalysisProblemData problemData = Problem.ProblemData as DataAnalysisProblemData;
648        if (problemData != null) {
649          problemData.TrainingPartitionParameter.Hidden = true;
650          problemData.TestPartitionParameter.Hidden = true;
651        }
652        ISymbolicDataAnalysisProblem symbolicProblem = Problem as ISymbolicDataAnalysisProblem;
653        if (symbolicProblem != null) {
654          symbolicProblem.FitnessCalculationPartitionParameter.Hidden = true;
655          symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
656          symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
657          symbolicProblem.ValidationPartitionParameter.Hidden = true;
658          symbolicProblem.ValidationPartition.Start = 0;
659          symbolicProblem.ValidationPartition.End = 0;
660        }
661      } else
662        SamplesEnd.Value = 0;
663    }
664
665    private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
666      switch (Algorithm.ExecutionState) {
667        case ExecutionState.Prepared:
668          OnPrepared();
669          break;
670        case ExecutionState.Started: throw new InvalidOperationException("Algorithm template can not be started.");
671        case ExecutionState.Paused: throw new InvalidOperationException("Algorithm template can not be paused.");
672        case ExecutionState.Stopped:
673          OnStopped();
674          break;
675      }
676    }
677    #endregion
678
679    #region clonedAlgorithms events
680    private void RegisterClonedAlgorithmsEvents() {
681      clonedAlgorithms.ItemsAdded += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
682      clonedAlgorithms.ItemsRemoved += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
683      clonedAlgorithms.CollectionReset += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
684      foreach (IAlgorithm algorithm in clonedAlgorithms)
685        RegisterClonedAlgorithmEvents(algorithm);
686    }
687    private void DeregisterClonedAlgorithmsEvents() {
688      clonedAlgorithms.ItemsAdded -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
689      clonedAlgorithms.ItemsRemoved -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
690      clonedAlgorithms.CollectionReset -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
691      foreach (IAlgorithm algorithm in clonedAlgorithms)
692        DeregisterClonedAlgorithmEvents(algorithm);
693    }
694    private void ClonedAlgorithms_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
695      foreach (IAlgorithm algorithm in e.Items)
696        RegisterClonedAlgorithmEvents(algorithm);
697    }
698    private void ClonedAlgorithms_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
699      foreach (IAlgorithm algorithm in e.Items)
700        DeregisterClonedAlgorithmEvents(algorithm);
701    }
702    private void ClonedAlgorithms_CollectionReset(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
703      foreach (IAlgorithm algorithm in e.OldItems)
704        DeregisterClonedAlgorithmEvents(algorithm);
705      foreach (IAlgorithm algorithm in e.Items)
706        RegisterClonedAlgorithmEvents(algorithm);
707    }
708    private void RegisterClonedAlgorithmEvents(IAlgorithm algorithm) {
709      algorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
710      algorithm.ExecutionTimeChanged += new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
711      algorithm.Started += new EventHandler(ClonedAlgorithm_Started);
712      algorithm.Paused += new EventHandler(ClonedAlgorithm_Paused);
713      algorithm.Stopped += new EventHandler(ClonedAlgorithm_Stopped);
714    }
715    private void DeregisterClonedAlgorithmEvents(IAlgorithm algorithm) {
716      algorithm.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
717      algorithm.ExecutionTimeChanged -= new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
718      algorithm.Started -= new EventHandler(ClonedAlgorithm_Started);
719      algorithm.Paused -= new EventHandler(ClonedAlgorithm_Paused);
720      algorithm.Stopped -= new EventHandler(ClonedAlgorithm_Stopped);
721    }
722    private void ClonedAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
723      Pause();
724      OnExceptionOccurred(e.Value);
725    }
726    private void ClonedAlgorithm_ExecutionTimeChanged(object sender, EventArgs e) {
727      OnExecutionTimeChanged();
728    }
729
730    private readonly object locker = new object();
731    private readonly object resultLocker = new object();
732    private void ClonedAlgorithm_Started(object sender, EventArgs e) {
733      IAlgorithm algorithm = sender as IAlgorithm;
734      lock (resultLocker) {
735        if (algorithm != null && !results.ContainsKey(algorithm.Name))
736          results.Add(new Result(algorithm.Name, "Contains results for the specific fold.", algorithm.Results));
737      }
738    }
739
740    private void ClonedAlgorithm_Paused(object sender, EventArgs e) {
741      lock (locker) {
742        availableWorkers.Release();
743        if (clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started)) {
744          OnPaused();
745          allAlgorithmsFinished.Set();
746        }
747      }
748    }
749
750    private void ClonedAlgorithm_Stopped(object sender, EventArgs e) {
751      lock (locker) {
752        // if the algorithm was in paused state, its worker has already been released
753        if (availableWorkers.CurrentCount < NumberOfWorkers.Value)
754          availableWorkers.Release();
755        if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped)) {
756          OnStopped();
757          allAlgorithmsFinished.Set();
758        } else if (stopPending && clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped)) {
759          OnStopped();
760          allAlgorithmsFinished.Set();
761        }
762      }
763    }
764    #endregion
765    #endregion
766
767    #region event firing
768    public event EventHandler ExecutionStateChanged;
769    private void OnExecutionStateChanged() {
770      EventHandler handler = ExecutionStateChanged;
771      if (handler != null) handler(this, EventArgs.Empty);
772    }
773    public event EventHandler ExecutionTimeChanged;
774    private void OnExecutionTimeChanged() {
775      EventHandler handler = ExecutionTimeChanged;
776      if (handler != null) handler(this, EventArgs.Empty);
777    }
778    public event EventHandler Prepared;
779    private void OnPrepared() {
780      ExecutionState = ExecutionState.Prepared;
781      EventHandler handler = Prepared;
782      if (handler != null) handler(this, EventArgs.Empty);
783      OnExecutionTimeChanged();
784    }
785    public event EventHandler Started;
786    private void OnStarted() {
787      startPending = false;
788      ExecutionState = ExecutionState.Started;
789      EventHandler handler = Started;
790      if (handler != null) handler(this, EventArgs.Empty);
791    }
792    public event EventHandler Paused;
793    private void OnPaused() {
794      pausePending = false;
795      ExecutionState = ExecutionState.Paused;
796      EventHandler handler = Paused;
797      if (handler != null) handler(this, EventArgs.Empty);
798    }
799    public event EventHandler Stopped;
800    private void OnStopped() {
801      stopPending = false;
802      Dictionary<string, IItem> collectedResults = new Dictionary<string, IItem>();
803      AggregateResultValues(collectedResults);
804      results.AddRange(collectedResults.Select(x => new Result(x.Key, x.Value)).Cast<IResult>().ToArray());
805      clonedAlgorithms.Clear();
806      runsCounter++;
807      runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
808      ExecutionState = ExecutionState.Stopped;
809      EventHandler handler = Stopped;
810      if (handler != null) handler(this, EventArgs.Empty);
811    }
812    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
813    private void OnExceptionOccurred(Exception exception) {
814      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
815      if (handler != null) handler(this, new EventArgs<Exception>(exception));
816    }
817    public event EventHandler StoreAlgorithmInEachRunChanged;
818    private void OnStoreAlgorithmInEachRunChanged() {
819      EventHandler handler = StoreAlgorithmInEachRunChanged;
820      if (handler != null) handler(this, EventArgs.Empty);
821    }
822    #endregion
823
824    #region helper
825
826    private void SetEstimationLimits(IDataAnalysisProblem problem, Type[] types) {
827      foreach (var type in types) {
828        if (type.IsAssignableFrom(problem.GetType())) {
829          var originalLimits = (DoubleLimit)Problem.Parameters["EstimationLimits"].ActualValue;  // problem is a clone of Problem
830          var limits = (DoubleLimit)problem.Parameters["EstimationLimits"].ActualValue;
831          limits.Lower = originalLimits.Lower;
832          limits.Upper = originalLimits.Upper;
833        }
834      }
835    }
836
837    #endregion
838  }
839}
Note: See TracBrowser for help on using the repository browser.