1 | #region License Information
|
---|
2 | /* HeuristicLab
|
---|
3 | * Copyright (C) 2002-2011 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
|
---|
4 | *
|
---|
5 | * This file is part of HeuristicLab.
|
---|
6 | *
|
---|
7 | * HeuristicLab is free software: you can redistribute it and/or modify
|
---|
8 | * it under the terms of the GNU General Public License as published by
|
---|
9 | * the Free Software Foundation, either version 3 of the License, or
|
---|
10 | * (at your option) any later version.
|
---|
11 | *
|
---|
12 | * HeuristicLab is distributed in the hope that it will be useful,
|
---|
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
---|
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
---|
15 | * GNU General Public License for more details.
|
---|
16 | *
|
---|
17 | * You should have received a copy of the GNU General Public License
|
---|
18 | * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
|
---|
19 | */
|
---|
20 | #endregion
|
---|
21 |
|
---|
22 | using System;
|
---|
23 | using System.Collections.Generic;
|
---|
24 | using System.Drawing;
|
---|
25 | using System.Linq;
|
---|
26 | using HeuristicLab.Collections;
|
---|
27 | using HeuristicLab.Common;
|
---|
28 | using HeuristicLab.Core;
|
---|
29 | using HeuristicLab.Data;
|
---|
30 | using HeuristicLab.Optimization;
|
---|
31 | using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
|
---|
32 | using HeuristicLab.Problems.DataAnalysis;
|
---|
33 |
|
---|
34 | namespace HeuristicLab.Algorithms.DataAnalysis {
|
---|
35 | [Item("Cross Validation", "Cross Validation wrapper for data analysis algorithms.")]
|
---|
36 | [Creatable("Data Analysis")]
|
---|
37 | [StorableClass]
|
---|
38 | public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent {
|
---|
39 | public CrossValidation()
|
---|
40 | : base() {
|
---|
41 | name = ItemName;
|
---|
42 | description = ItemDescription;
|
---|
43 |
|
---|
44 | executionState = ExecutionState.Stopped;
|
---|
45 | runs = new RunCollection();
|
---|
46 | runsCounter = 0;
|
---|
47 |
|
---|
48 | algorithm = null;
|
---|
49 | clonedAlgorithms = new ItemCollection<IAlgorithm>();
|
---|
50 | results = new ResultCollection();
|
---|
51 |
|
---|
52 | folds = new IntValue(2);
|
---|
53 | numberOfWorkers = new IntValue(1);
|
---|
54 | samplesStart = new IntValue(0);
|
---|
55 | samplesEnd = new IntValue(0);
|
---|
56 | storeAlgorithmInEachRun = false;
|
---|
57 |
|
---|
58 | RegisterEvents();
|
---|
59 | if (Algorithm != null) RegisterAlgorithmEvents();
|
---|
60 | }
|
---|
61 |
|
---|
62 | public string Filename { get; set; }
|
---|
63 |
|
---|
64 | #region persistence and cloning
|
---|
65 | [StorableConstructor]
|
---|
66 | private CrossValidation(bool deserializing)
|
---|
67 | : base(deserializing) {
|
---|
68 | }
|
---|
69 | [StorableHook(HookType.AfterDeserialization)]
|
---|
70 | private void AfterDeserialization() {
|
---|
71 | RegisterEvents();
|
---|
72 | if (Algorithm != null) RegisterAlgorithmEvents();
|
---|
73 | }
|
---|
74 |
|
---|
75 | private CrossValidation(CrossValidation original, Cloner cloner)
|
---|
76 | : base(original, cloner) {
|
---|
77 | executionState = original.executionState;
|
---|
78 | storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
|
---|
79 | runs = cloner.Clone(original.runs);
|
---|
80 | runsCounter = original.runsCounter;
|
---|
81 | algorithm = cloner.Clone(original.algorithm);
|
---|
82 | clonedAlgorithms = cloner.Clone(original.clonedAlgorithms);
|
---|
83 | results = cloner.Clone(original.results);
|
---|
84 |
|
---|
85 | folds = cloner.Clone(original.folds);
|
---|
86 | numberOfWorkers = cloner.Clone(original.numberOfWorkers);
|
---|
87 | samplesStart = cloner.Clone(original.samplesStart);
|
---|
88 | samplesEnd = cloner.Clone(original.samplesEnd);
|
---|
89 | RegisterEvents();
|
---|
90 | if (Algorithm != null) RegisterAlgorithmEvents();
|
---|
91 | }
|
---|
92 | public override IDeepCloneable Clone(Cloner cloner) {
|
---|
93 | return new CrossValidation(this, cloner);
|
---|
94 | }
|
---|
95 |
|
---|
96 | #endregion
|
---|
97 |
|
---|
98 | #region properties
|
---|
99 | [Storable]
|
---|
100 | private IAlgorithm algorithm;
|
---|
101 | public IAlgorithm Algorithm {
|
---|
102 | get { return algorithm; }
|
---|
103 | set {
|
---|
104 | if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
|
---|
105 | throw new InvalidOperationException("Changing the algorithm is only allowed if the CrossValidation is stopped or prepared.");
|
---|
106 | if (algorithm != value) {
|
---|
107 | if (value != null && value.Problem != null && !(value.Problem is IDataAnalysisProblem<IDataAnalysisProblemData>))
|
---|
108 | throw new ArgumentException("Only algorithms with a DataAnalysisProblem could be used for the cross validation.");
|
---|
109 | if (algorithm != null) DeregisterAlgorithmEvents();
|
---|
110 | algorithm = value;
|
---|
111 | Parameters.Clear();
|
---|
112 |
|
---|
113 | if (algorithm != null) {
|
---|
114 | algorithm.StoreAlgorithmInEachRun = false;
|
---|
115 | RegisterAlgorithmEvents();
|
---|
116 | algorithm.Prepare(true);
|
---|
117 | Parameters.AddRange(algorithm.Parameters);
|
---|
118 | }
|
---|
119 | OnAlgorithmChanged();
|
---|
120 | if (algorithm != null) OnProblemChanged();
|
---|
121 | Prepare();
|
---|
122 | }
|
---|
123 | }
|
---|
124 | }
|
---|
125 |
|
---|
126 |
|
---|
127 | [Storable]
|
---|
128 | private IDataAnalysisProblem<IDataAnalysisProblemData> problem;
|
---|
129 | public IDataAnalysisProblem<IDataAnalysisProblemData> Problem {
|
---|
130 | get {
|
---|
131 | if (algorithm == null)
|
---|
132 | return null;
|
---|
133 | return (IDataAnalysisProblem<IDataAnalysisProblemData>)algorithm.Problem;
|
---|
134 | }
|
---|
135 | set {
|
---|
136 | if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
|
---|
137 | throw new InvalidOperationException("Changing the problem is only allowed if the CrossValidation is stopped or prepared.");
|
---|
138 | if (algorithm == null) throw new ArgumentNullException("Could not set a problem before an algorithm was set.");
|
---|
139 | algorithm.Problem = value;
|
---|
140 | problem = value;
|
---|
141 | }
|
---|
142 | }
|
---|
143 |
|
---|
144 | IProblem IAlgorithm.Problem {
|
---|
145 | get { return Problem; }
|
---|
146 | set {
|
---|
147 | if (value != null && !ProblemType.IsInstanceOfType(value))
|
---|
148 | throw new ArgumentException("Only DataAnalysisProblems could be used for the cross validation.");
|
---|
149 | Problem = (IDataAnalysisProblem<IDataAnalysisProblemData>)value;
|
---|
150 | }
|
---|
151 | }
|
---|
152 | public Type ProblemType {
|
---|
153 | get { return typeof(IDataAnalysisProblem<IDataAnalysisProblemData>); }
|
---|
154 | }
|
---|
155 |
|
---|
156 | [Storable]
|
---|
157 | private ItemCollection<IAlgorithm> clonedAlgorithms;
|
---|
158 |
|
---|
159 | public IEnumerable<IOptimizer> NestedOptimizers {
|
---|
160 | get {
|
---|
161 | if (Algorithm == null) yield break;
|
---|
162 | yield return Algorithm;
|
---|
163 | }
|
---|
164 | }
|
---|
165 |
|
---|
166 | [Storable]
|
---|
167 | private ResultCollection results;
|
---|
168 | public ResultCollection Results {
|
---|
169 | get { return results; }
|
---|
170 | }
|
---|
171 |
|
---|
172 | [Storable]
|
---|
173 | private IntValue folds;
|
---|
174 | public IntValue Folds {
|
---|
175 | get { return folds; }
|
---|
176 | }
|
---|
177 | [Storable]
|
---|
178 | private IntValue samplesStart;
|
---|
179 | public IntValue SamplesStart {
|
---|
180 | get { return samplesStart; }
|
---|
181 | }
|
---|
182 | [Storable]
|
---|
183 | private IntValue samplesEnd;
|
---|
184 | public IntValue SamplesEnd {
|
---|
185 | get { return samplesEnd; }
|
---|
186 | }
|
---|
187 | [Storable]
|
---|
188 | private IntValue numberOfWorkers;
|
---|
189 | public IntValue NumberOfWorkers {
|
---|
190 | get { return numberOfWorkers; }
|
---|
191 | }
|
---|
192 |
|
---|
193 | [Storable]
|
---|
194 | private bool storeAlgorithmInEachRun;
|
---|
195 | public bool StoreAlgorithmInEachRun {
|
---|
196 | get { return storeAlgorithmInEachRun; }
|
---|
197 | set {
|
---|
198 | if (storeAlgorithmInEachRun != value) {
|
---|
199 | storeAlgorithmInEachRun = value;
|
---|
200 | OnStoreAlgorithmInEachRunChanged();
|
---|
201 | }
|
---|
202 | }
|
---|
203 | }
|
---|
204 |
|
---|
205 | [Storable]
|
---|
206 | private int runsCounter;
|
---|
207 | [Storable]
|
---|
208 | private RunCollection runs;
|
---|
209 | public RunCollection Runs {
|
---|
210 | get { return runs; }
|
---|
211 | }
|
---|
212 |
|
---|
213 | [Storable]
|
---|
214 | private ExecutionState executionState;
|
---|
215 | public ExecutionState ExecutionState {
|
---|
216 | get { return executionState; }
|
---|
217 | private set {
|
---|
218 | if (executionState != value) {
|
---|
219 | executionState = value;
|
---|
220 | OnExecutionStateChanged();
|
---|
221 | OnItemImageChanged();
|
---|
222 | }
|
---|
223 | }
|
---|
224 | }
|
---|
225 | public override Image ItemImage {
|
---|
226 | get {
|
---|
227 | if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
|
---|
228 | else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
|
---|
229 | else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
|
---|
230 | else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
|
---|
231 | else return HeuristicLab.Common.Resources.VSImageLibrary.Event;
|
---|
232 | }
|
---|
233 | }
|
---|
234 |
|
---|
235 | public TimeSpan ExecutionTime {
|
---|
236 | get {
|
---|
237 | if (ExecutionState != ExecutionState.Prepared)
|
---|
238 | return TimeSpan.FromMilliseconds(clonedAlgorithms.Select(x => x.ExecutionTime.TotalMilliseconds).Sum());
|
---|
239 | return TimeSpan.Zero;
|
---|
240 | }
|
---|
241 | }
|
---|
242 | #endregion
|
---|
243 |
|
---|
244 | public void Prepare() {
|
---|
245 | if (ExecutionState == ExecutionState.Started)
|
---|
246 | throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
|
---|
247 | results.Clear();
|
---|
248 | clonedAlgorithms.Clear();
|
---|
249 | if (Algorithm != null) {
|
---|
250 | Algorithm.Prepare();
|
---|
251 | if (Algorithm.ExecutionState == ExecutionState.Prepared) OnPrepared();
|
---|
252 | }
|
---|
253 | }
|
---|
254 | public void Prepare(bool clearRuns) {
|
---|
255 | if (clearRuns) runs.Clear();
|
---|
256 | Prepare();
|
---|
257 | }
|
---|
258 |
|
---|
259 | private bool startPending;
|
---|
260 | public void Start() {
|
---|
261 | if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
|
---|
262 | throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));
|
---|
263 |
|
---|
264 | if (Algorithm != null && !startPending) {
|
---|
265 | startPending = true;
|
---|
266 | //create cloned algorithms
|
---|
267 | if (clonedAlgorithms.Count == 0) {
|
---|
268 | int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / Folds.Value;
|
---|
269 | for (int i = 0; i < Folds.Value; i++) {
|
---|
270 | IAlgorithm clonedAlgorithm = (IAlgorithm)algorithm.Clone();
|
---|
271 | clonedAlgorithm.Name = algorithm.Name + " Fold " + i;
|
---|
272 | IDataAnalysisProblem<IDataAnalysisProblemData> problem = clonedAlgorithm.Problem as IDataAnalysisProblem<IDataAnalysisProblemData>;
|
---|
273 | problem.ProblemData.TestPartitionEnd.Value = (i + 1) == Folds.Value ? SamplesEnd.Value : (i + 1) * testSamplesCount + SamplesStart.Value;
|
---|
274 | problem.ProblemData.TestPartitionStart.Value = (i * testSamplesCount) + SamplesStart.Value;
|
---|
275 | clonedAlgorithms.Add(clonedAlgorithm);
|
---|
276 | }
|
---|
277 | }
|
---|
278 |
|
---|
279 | //start prepared or paused cloned algorithms
|
---|
280 | int startedAlgorithms = 0;
|
---|
281 | foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
|
---|
282 | if (startedAlgorithms < NumberOfWorkers.Value) {
|
---|
283 | if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared ||
|
---|
284 | clonedAlgorithm.ExecutionState == ExecutionState.Paused) {
|
---|
285 | clonedAlgorithm.Start();
|
---|
286 | startedAlgorithms++;
|
---|
287 | }
|
---|
288 | }
|
---|
289 | }
|
---|
290 | OnStarted();
|
---|
291 | }
|
---|
292 | }
|
---|
293 |
|
---|
294 | private bool pausePending;
|
---|
295 | public void Pause() {
|
---|
296 | if (ExecutionState != ExecutionState.Started)
|
---|
297 | throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
|
---|
298 | if (!pausePending) {
|
---|
299 | pausePending = true;
|
---|
300 | if (!startPending) PauseAllClonedAlgorithms();
|
---|
301 | }
|
---|
302 | }
|
---|
303 | private void PauseAllClonedAlgorithms() {
|
---|
304 | foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
|
---|
305 | if (clonedAlgorithm.ExecutionState == ExecutionState.Started)
|
---|
306 | clonedAlgorithm.Pause();
|
---|
307 | }
|
---|
308 | }
|
---|
309 |
|
---|
310 | private bool stopPending;
|
---|
311 | public void Stop() {
|
---|
312 | if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
|
---|
313 | throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".",
|
---|
314 | ExecutionState));
|
---|
315 | if (!stopPending) {
|
---|
316 | stopPending = true;
|
---|
317 | if (!startPending) StopAllClonedAlgorithms();
|
---|
318 | }
|
---|
319 | }
|
---|
320 | private void StopAllClonedAlgorithms() {
|
---|
321 | foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
|
---|
322 | if (clonedAlgorithm.ExecutionState == ExecutionState.Started ||
|
---|
323 | clonedAlgorithm.ExecutionState == ExecutionState.Paused)
|
---|
324 | clonedAlgorithm.Stop();
|
---|
325 | }
|
---|
326 | }
|
---|
327 |
|
---|
328 | #region collect parameters and results
|
---|
329 | public override void CollectParameterValues(IDictionary<string, IItem> values) {
|
---|
330 | values.Add("Algorithm Name", new StringValue(Name));
|
---|
331 | values.Add("Algorithm Type", new StringValue(GetType().GetPrettyName()));
|
---|
332 | values.Add("Folds", new IntValue(Folds.Value));
|
---|
333 |
|
---|
334 | if (algorithm != null) {
|
---|
335 | values.Add("CrossValidation Algorithm Name", new StringValue(Algorithm.Name));
|
---|
336 | values.Add("CrossValidation Algorithm Type", new StringValue(Algorithm.GetType().GetPrettyName()));
|
---|
337 | base.CollectParameterValues(values);
|
---|
338 | }
|
---|
339 | if (Problem != null) {
|
---|
340 | values.Add("Problem Name", new StringValue(Problem.Name));
|
---|
341 | values.Add("Problem Type", new StringValue(Problem.GetType().GetPrettyName()));
|
---|
342 | Problem.CollectParameterValues(values);
|
---|
343 | }
|
---|
344 | }
|
---|
345 |
|
---|
346 | public void CollectResultValues(IDictionary<string, IItem> results) {
|
---|
347 | Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
|
---|
348 | IEnumerable<IRun> runs = clonedAlgorithms.Select(alg => alg.Runs.FirstOrDefault()).Where(run => run != null);
|
---|
349 | IEnumerable<KeyValuePair<string, IItem>> resultCollections = runs.Where(x => x != null).SelectMany(x => x.Results).ToList();
|
---|
350 |
|
---|
351 | foreach (IResult result in ExtractAndAggregateResults<IntValue>(resultCollections))
|
---|
352 | results.Add(result.Name, result.Value);
|
---|
353 | foreach (IResult result in ExtractAndAggregateResults<DoubleValue>(resultCollections))
|
---|
354 | results.Add(result.Name, result.Value);
|
---|
355 | foreach (IResult result in ExtractAndAggregateResults<PercentValue>(resultCollections))
|
---|
356 | results.Add(result.Name, result.Value);
|
---|
357 |
|
---|
358 | results.Add("Execution Time", new TimeSpanValue(this.ExecutionTime));
|
---|
359 | results.Add("CrossValidation Folds", new RunCollection(runs));
|
---|
360 | }
|
---|
361 |
|
---|
362 | private static IEnumerable<IResult> ExtractAndAggregateResults<T>(IEnumerable<KeyValuePair<string, IItem>> results)
|
---|
363 | where T : class, IItem, new() {
|
---|
364 | Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
|
---|
365 | foreach (var resultValue in results.Where(r => r.Value.GetType() == typeof(T))) {
|
---|
366 | if (!resultValues.ContainsKey(resultValue.Key))
|
---|
367 | resultValues[resultValue.Key] = new List<double>();
|
---|
368 | resultValues[resultValue.Key].Add(ConvertToDouble(resultValue.Value));
|
---|
369 | }
|
---|
370 |
|
---|
371 | DoubleValue doubleValue;
|
---|
372 | if (typeof(T) == typeof(PercentValue))
|
---|
373 | doubleValue = new PercentValue();
|
---|
374 | else if (typeof(T) == typeof(DoubleValue))
|
---|
375 | doubleValue = new DoubleValue();
|
---|
376 | else if (typeof(T) == typeof(IntValue))
|
---|
377 | doubleValue = new DoubleValue();
|
---|
378 | else
|
---|
379 | throw new NotSupportedException();
|
---|
380 |
|
---|
381 | List<IResult> aggregatedResults = new List<IResult>();
|
---|
382 | foreach (KeyValuePair<string, List<double>> resultValue in resultValues) {
|
---|
383 | doubleValue.Value = resultValue.Value.Average();
|
---|
384 | aggregatedResults.Add(new Result(resultValue.Key, (IItem)doubleValue.Clone()));
|
---|
385 | doubleValue.Value = resultValue.Value.StandardDeviation();
|
---|
386 | aggregatedResults.Add(new Result(resultValue.Key + " StdDev", (IItem)doubleValue.Clone()));
|
---|
387 | }
|
---|
388 | return aggregatedResults;
|
---|
389 | }
|
---|
390 |
|
---|
391 | private static double ConvertToDouble(IItem item) {
|
---|
392 | if (item is DoubleValue) return ((DoubleValue)item).Value;
|
---|
393 | else if (item is IntValue) return ((IntValue)item).Value;
|
---|
394 | else throw new NotSupportedException("Could not convert any item type to double");
|
---|
395 | }
|
---|
396 | #endregion
|
---|
397 |
|
---|
398 | #region events
|
---|
399 | private void RegisterEvents() {
|
---|
400 | Folds.ValueChanged += new EventHandler(Folds_ValueChanged);
|
---|
401 | SamplesStart.ValueChanged += new EventHandler(SamplesStart_ValueChanged);
|
---|
402 | SamplesEnd.ValueChanged += new EventHandler(SamplesEnd_ValueChanged);
|
---|
403 | RegisterClonedAlgorithmsEvents();
|
---|
404 | }
|
---|
405 | private void Folds_ValueChanged(object sender, EventArgs e) {
|
---|
406 | if (ExecutionState != ExecutionState.Prepared)
|
---|
407 | throw new InvalidOperationException("Can not change number of folds if the execution state is not prepared.");
|
---|
408 | }
|
---|
409 | private void SamplesStart_ValueChanged(object sender, EventArgs e) {
|
---|
410 | if (Problem != null) Problem.ProblemData.TrainingPartitionStart.Value = SamplesStart.Value;
|
---|
411 | }
|
---|
412 | private void SamplesEnd_ValueChanged(object sender, EventArgs e) {
|
---|
413 | if (Problem != null) Problem.ProblemData.TrainingPartitionEnd.Value = SamplesEnd.Value;
|
---|
414 | }
|
---|
415 |
|
---|
416 | #region template algorithms events
|
---|
417 | public event EventHandler AlgorithmChanged;
|
---|
418 | private void OnAlgorithmChanged() {
|
---|
419 | EventHandler handler = AlgorithmChanged;
|
---|
420 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
421 | OnProblemChanged();
|
---|
422 | if (Problem == null) ExecutionState = ExecutionState.Stopped;
|
---|
423 | }
|
---|
424 | private void RegisterAlgorithmEvents() {
|
---|
425 | algorithm.ProblemChanged += new EventHandler(Algorithm_ProblemChanged);
|
---|
426 | algorithm.ExecutionStateChanged += new EventHandler(Algorithm_ExecutionStateChanged);
|
---|
427 | }
|
---|
428 | private void DeregisterAlgorithmEvents() {
|
---|
429 | algorithm.ProblemChanged -= new EventHandler(Algorithm_ProblemChanged);
|
---|
430 | algorithm.ExecutionStateChanged -= new EventHandler(Algorithm_ExecutionStateChanged);
|
---|
431 | }
|
---|
432 | private void Algorithm_ProblemChanged(object sender, EventArgs e) {
|
---|
433 | if (algorithm.Problem != null && !(algorithm.Problem is IDataAnalysisProblem<IDataAnalysisProblemData>)) {
|
---|
434 | algorithm.Problem = problem;
|
---|
435 | throw new ArgumentException("A cross validation algorithm can only contain DataAnalysisProblems.");
|
---|
436 | }
|
---|
437 | problem = (IDataAnalysisProblem<IDataAnalysisProblemData>)algorithm.Problem;
|
---|
438 | OnProblemChanged();
|
---|
439 | }
|
---|
440 | public event EventHandler ProblemChanged;
|
---|
441 | private void OnProblemChanged() {
|
---|
442 | EventHandler handler = ProblemChanged;
|
---|
443 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
444 |
|
---|
445 | SamplesStart.Value = 0;
|
---|
446 | if (Problem != null)
|
---|
447 | SamplesEnd.Value = Problem.ProblemData.Dataset.Rows;
|
---|
448 | else
|
---|
449 | SamplesEnd.Value = 0;
|
---|
450 | }
|
---|
451 |
|
---|
452 | private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
|
---|
453 | switch (Algorithm.ExecutionState) {
|
---|
454 | case ExecutionState.Prepared: OnPrepared();
|
---|
455 | break;
|
---|
456 | case ExecutionState.Started: throw new InvalidOperationException("Algorithm template can not be started.");
|
---|
457 | case ExecutionState.Paused: throw new InvalidOperationException("Algorithm template can not be paused.");
|
---|
458 | case ExecutionState.Stopped: OnStopped();
|
---|
459 | break;
|
---|
460 | }
|
---|
461 | }
|
---|
462 | #endregion
|
---|
463 |
|
---|
464 | #region clonedAlgorithms events
|
---|
465 | private void RegisterClonedAlgorithmsEvents() {
|
---|
466 | clonedAlgorithms.ItemsAdded += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
|
---|
467 | clonedAlgorithms.ItemsRemoved += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
|
---|
468 | clonedAlgorithms.CollectionReset += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
|
---|
469 | foreach (IAlgorithm algorithm in clonedAlgorithms)
|
---|
470 | RegisterClonedAlgorithmEvents(algorithm);
|
---|
471 | }
|
---|
472 | private void DeregisterClonedAlgorithmsEvents() {
|
---|
473 | clonedAlgorithms.ItemsAdded -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
|
---|
474 | clonedAlgorithms.ItemsRemoved -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
|
---|
475 | clonedAlgorithms.CollectionReset -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
|
---|
476 | foreach (IAlgorithm algorithm in clonedAlgorithms)
|
---|
477 | DeregisterClonedAlgorithmEvents(algorithm);
|
---|
478 | }
|
---|
479 | private void ClonedAlgorithms_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
|
---|
480 | foreach (IAlgorithm algorithm in e.Items)
|
---|
481 | RegisterClonedAlgorithmEvents(algorithm);
|
---|
482 | }
|
---|
483 | private void ClonedAlgorithms_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
|
---|
484 | foreach (IAlgorithm algorithm in e.Items)
|
---|
485 | DeregisterClonedAlgorithmEvents(algorithm);
|
---|
486 | }
|
---|
487 | private void ClonedAlgorithms_CollectionReset(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
|
---|
488 | foreach (IAlgorithm algorithm in e.OldItems)
|
---|
489 | DeregisterClonedAlgorithmEvents(algorithm);
|
---|
490 | foreach (IAlgorithm algorithm in e.Items)
|
---|
491 | RegisterClonedAlgorithmEvents(algorithm);
|
---|
492 | }
|
---|
493 | private void RegisterClonedAlgorithmEvents(IAlgorithm algorithm) {
|
---|
494 | algorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
|
---|
495 | algorithm.ExecutionTimeChanged += new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
|
---|
496 | algorithm.Started += new EventHandler(ClonedAlgorithm_Started);
|
---|
497 | algorithm.Paused += new EventHandler(ClonedAlgorithm_Paused);
|
---|
498 | algorithm.Stopped += new EventHandler(ClonedAlgorithm_Stopped);
|
---|
499 | }
|
---|
500 | private void DeregisterClonedAlgorithmEvents(IAlgorithm algorithm) {
|
---|
501 | algorithm.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
|
---|
502 | algorithm.ExecutionTimeChanged -= new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
|
---|
503 | algorithm.Started -= new EventHandler(ClonedAlgorithm_Started);
|
---|
504 | algorithm.Paused -= new EventHandler(ClonedAlgorithm_Paused);
|
---|
505 | algorithm.Stopped -= new EventHandler(ClonedAlgorithm_Stopped);
|
---|
506 | }
|
---|
507 | private void ClonedAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
|
---|
508 | OnExceptionOccurred(e.Value);
|
---|
509 | }
|
---|
510 | private void ClonedAlgorithm_ExecutionTimeChanged(object sender, EventArgs e) {
|
---|
511 | OnExecutionTimeChanged();
|
---|
512 | }
|
---|
513 |
|
---|
514 | private readonly object locker = new object();
|
---|
515 | private void ClonedAlgorithm_Started(object sender, EventArgs e) {
|
---|
516 | lock (locker) {
|
---|
517 | IAlgorithm algorithm = sender as IAlgorithm;
|
---|
518 | if (algorithm != null && !results.ContainsKey(algorithm.Name))
|
---|
519 | results.Add(new Result(algorithm.Name, "Contains results for the specific fold.", algorithm.Results));
|
---|
520 |
|
---|
521 | if (startPending) {
|
---|
522 | int startedAlgorithms = clonedAlgorithms.Count(alg => alg.ExecutionState == ExecutionState.Started);
|
---|
523 | if (startedAlgorithms == NumberOfWorkers.Value ||
|
---|
524 | clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Prepared))
|
---|
525 | startPending = false;
|
---|
526 |
|
---|
527 | if (pausePending) PauseAllClonedAlgorithms();
|
---|
528 | if (stopPending) StopAllClonedAlgorithms();
|
---|
529 | }
|
---|
530 | }
|
---|
531 | }
|
---|
532 |
|
---|
533 | private void ClonedAlgorithm_Paused(object sender, EventArgs e) {
|
---|
534 | lock (locker) {
|
---|
535 | if (pausePending && clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started))
|
---|
536 | OnPaused();
|
---|
537 | }
|
---|
538 | }
|
---|
539 |
|
---|
540 | private void ClonedAlgorithm_Stopped(object sender, EventArgs e) {
|
---|
541 | lock (locker) {
|
---|
542 | if (!stopPending && ExecutionState == ExecutionState.Started) {
|
---|
543 | IAlgorithm preparedAlgorithm = clonedAlgorithms.Where(alg => alg.ExecutionState == ExecutionState.Prepared ||
|
---|
544 | alg.ExecutionState == ExecutionState.Paused).FirstOrDefault();
|
---|
545 | if (preparedAlgorithm != null) preparedAlgorithm.Start();
|
---|
546 | }
|
---|
547 | if (ExecutionState != ExecutionState.Stopped) {
|
---|
548 | if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped))
|
---|
549 | OnStopped();
|
---|
550 | else if (stopPending &&
|
---|
551 | clonedAlgorithms.All(
|
---|
552 | alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped))
|
---|
553 | OnStopped();
|
---|
554 | }
|
---|
555 | }
|
---|
556 | }
|
---|
557 | #endregion
|
---|
558 | #endregion
|
---|
559 |
|
---|
560 | #region event firing
|
---|
561 | public event EventHandler ExecutionStateChanged;
|
---|
562 | private void OnExecutionStateChanged() {
|
---|
563 | EventHandler handler = ExecutionStateChanged;
|
---|
564 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
565 | }
|
---|
566 | public event EventHandler ExecutionTimeChanged;
|
---|
567 | private void OnExecutionTimeChanged() {
|
---|
568 | EventHandler handler = ExecutionTimeChanged;
|
---|
569 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
570 | }
|
---|
571 | public event EventHandler Prepared;
|
---|
572 | private void OnPrepared() {
|
---|
573 | ExecutionState = ExecutionState.Prepared;
|
---|
574 | EventHandler handler = Prepared;
|
---|
575 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
576 | OnExecutionTimeChanged();
|
---|
577 | }
|
---|
578 | public event EventHandler Started;
|
---|
579 | private void OnStarted() {
|
---|
580 | startPending = false;
|
---|
581 | ExecutionState = ExecutionState.Started;
|
---|
582 | EventHandler handler = Started;
|
---|
583 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
584 | }
|
---|
585 | public event EventHandler Paused;
|
---|
586 | private void OnPaused() {
|
---|
587 | pausePending = false;
|
---|
588 | ExecutionState = ExecutionState.Paused;
|
---|
589 | EventHandler handler = Paused;
|
---|
590 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
591 | }
|
---|
592 | public event EventHandler Stopped;
|
---|
593 | private void OnStopped() {
|
---|
594 | stopPending = false;
|
---|
595 | Dictionary<string, IItem> collectedResults = new Dictionary<string, IItem>();
|
---|
596 | CollectResultValues(collectedResults);
|
---|
597 | results.AddRange(collectedResults.Select(x => new Result(x.Key, x.Value)).Cast<IResult>().ToArray());
|
---|
598 | runsCounter++;
|
---|
599 | runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
|
---|
600 | ExecutionState = ExecutionState.Stopped;
|
---|
601 | EventHandler handler = Stopped;
|
---|
602 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
603 | }
|
---|
604 | public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
|
---|
605 | private void OnExceptionOccurred(Exception exception) {
|
---|
606 | EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
|
---|
607 | if (handler != null) handler(this, new EventArgs<Exception>(exception));
|
---|
608 | }
|
---|
609 | public event EventHandler StoreAlgorithmInEachRunChanged;
|
---|
610 | private void OnStoreAlgorithmInEachRunChanged() {
|
---|
611 | EventHandler handler = StoreAlgorithmInEachRunChanged;
|
---|
612 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
613 | }
|
---|
614 | #endregion
|
---|
615 | }
|
---|
616 | }
|
---|