[5617] | 1 | #region License Information
|
---|
| 2 | /* HeuristicLab
|
---|
[16565] | 3 | * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
|
---|
[5617] | 4 | *
|
---|
| 5 | * This file is part of HeuristicLab.
|
---|
| 6 | *
|
---|
| 7 | * HeuristicLab is free software: you can redistribute it and/or modify
|
---|
| 8 | * it under the terms of the GNU General Public License as published by
|
---|
| 9 | * the Free Software Foundation, either version 3 of the License, or
|
---|
| 10 | * (at your option) any later version.
|
---|
| 11 | *
|
---|
| 12 | * HeuristicLab is distributed in the hope that it will be useful,
|
---|
| 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
---|
| 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
---|
| 15 | * GNU General Public License for more details.
|
---|
| 16 | *
|
---|
| 17 | * You should have received a copy of the GNU General Public License
|
---|
| 18 | * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
|
---|
| 19 | */
|
---|
| 20 | #endregion
|
---|
| 21 |
|
---|
| 22 | using System;
|
---|
| 23 | using System.Collections.Generic;
|
---|
| 24 | using System.Drawing;
|
---|
| 25 | using System.Linq;
|
---|
[8836] | 26 | using System.Threading;
|
---|
[15287] | 27 | using System.Threading.Tasks;
|
---|
[5617] | 28 | using HeuristicLab.Collections;
|
---|
| 29 | using HeuristicLab.Common;
|
---|
| 30 | using HeuristicLab.Core;
|
---|
| 31 | using HeuristicLab.Data;
|
---|
| 32 | using HeuristicLab.Optimization;
|
---|
[16565] | 33 | using HEAL.Attic;
|
---|
[5617] | 34 | using HeuristicLab.Problems.DataAnalysis;
|
---|
[5886] | 35 | using HeuristicLab.Problems.DataAnalysis.Symbolic;
|
---|
[14864] | 36 | using HeuristicLab.Random;
|
---|
[5617] | 37 |
|
---|
| 38 | namespace HeuristicLab.Algorithms.DataAnalysis {
|
---|
[13238] | 39 | [Item("Cross Validation (CV)", "Cross-validation wrapper for data analysis algorithms.")]
|
---|
[12504] | 40 | [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 100)]
|
---|
[16565] | 41 | [StorableType("1C622121-AE5B-42FD-831C-FCA8F8E0AF8D")]
|
---|
[5617] | 42 | public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent {
|
---|
[15002] | 43 | [Storable]
|
---|
| 44 | private int seed;
|
---|
[14904] | 45 |
|
---|
[15287] | 46 | private SemaphoreSlim availableWorkers; // limits the number of concurrent algorithm executions
|
---|
| 47 | private ManualResetEventSlim allAlgorithmsFinished; // this indicates that all started algorithms have been paused or stopped
|
---|
| 48 |
|
---|
[5617] | 49 | public CrossValidation()
|
---|
| 50 | : base() {
|
---|
| 51 | name = ItemName;
|
---|
| 52 | description = ItemDescription;
|
---|
| 53 |
|
---|
| 54 | executionState = ExecutionState.Stopped;
|
---|
[8962] | 55 | runs = new RunCollection { OptimizerName = name };
|
---|
[5617] | 56 | runsCounter = 0;
|
---|
| 57 |
|
---|
| 58 | algorithm = null;
|
---|
| 59 | clonedAlgorithms = new ItemCollection<IAlgorithm>();
|
---|
| 60 | results = new ResultCollection();
|
---|
| 61 |
|
---|
| 62 | folds = new IntValue(2);
|
---|
| 63 | numberOfWorkers = new IntValue(1);
|
---|
| 64 | samplesStart = new IntValue(0);
|
---|
| 65 | samplesEnd = new IntValue(0);
|
---|
[14864] | 66 | shuffleSamples = new BoolValue(false);
|
---|
[5617] | 67 | storeAlgorithmInEachRun = false;
|
---|
| 68 |
|
---|
| 69 | RegisterEvents();
|
---|
| 70 | if (Algorithm != null) RegisterAlgorithmEvents();
|
---|
| 71 | }
|
---|
| 72 |
|
---|
| 73 | public string Filename { get; set; }
|
---|
| 74 |
|
---|
| 75 | #region persistence and cloning
|
---|
| 76 | [StorableConstructor]
|
---|
[16565] | 77 | private CrossValidation(StorableConstructorFlag _) : base(_) {
|
---|
[5617] | 78 | }
|
---|
| 79 | [StorableHook(HookType.AfterDeserialization)]
|
---|
| 80 | private void AfterDeserialization() {
|
---|
[15077] | 81 | // BackwardsCompatibility3.3
|
---|
| 82 | #region Backwards compatible code, remove with 3.4
|
---|
| 83 | if (shuffleSamples == null) shuffleSamples = new BoolValue(false);
|
---|
| 84 | #endregion
|
---|
| 85 |
|
---|
[5617] | 86 | RegisterEvents();
|
---|
| 87 | if (Algorithm != null) RegisterAlgorithmEvents();
|
---|
| 88 | }
|
---|
| 89 |
|
---|
| 90 | private CrossValidation(CrossValidation original, Cloner cloner)
|
---|
| 91 | : base(original, cloner) {
|
---|
| 92 | executionState = original.executionState;
|
---|
| 93 | storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
|
---|
| 94 | runs = cloner.Clone(original.runs);
|
---|
| 95 | runsCounter = original.runsCounter;
|
---|
| 96 | algorithm = cloner.Clone(original.algorithm);
|
---|
| 97 | clonedAlgorithms = cloner.Clone(original.clonedAlgorithms);
|
---|
| 98 | results = cloner.Clone(original.results);
|
---|
| 99 |
|
---|
| 100 | folds = cloner.Clone(original.folds);
|
---|
| 101 | numberOfWorkers = cloner.Clone(original.numberOfWorkers);
|
---|
| 102 | samplesStart = cloner.Clone(original.samplesStart);
|
---|
| 103 | samplesEnd = cloner.Clone(original.samplesEnd);
|
---|
[14864] | 104 | shuffleSamples = cloner.Clone(original.shuffleSamples);
|
---|
[15002] | 105 | seed = original.seed;
|
---|
| 106 |
|
---|
[5617] | 107 | RegisterEvents();
|
---|
| 108 | if (Algorithm != null) RegisterAlgorithmEvents();
|
---|
| 109 | }
|
---|
| 110 | public override IDeepCloneable Clone(Cloner cloner) {
|
---|
| 111 | return new CrossValidation(this, cloner);
|
---|
| 112 | }
|
---|
| 113 |
|
---|
| 114 | #endregion
|
---|
| 115 |
|
---|
| 116 | #region properties
|
---|
| 117 | [Storable]
|
---|
| 118 | private IAlgorithm algorithm;
|
---|
| 119 | public IAlgorithm Algorithm {
|
---|
| 120 | get { return algorithm; }
|
---|
| 121 | set {
|
---|
| 122 | if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
|
---|
| 123 | throw new InvalidOperationException("Changing the algorithm is only allowed if the CrossValidation is stopped or prepared.");
|
---|
| 124 | if (algorithm != value) {
|
---|
[5659] | 125 | if (value != null && value.Problem != null && !(value.Problem is IDataAnalysisProblem))
|
---|
[5617] | 126 | throw new ArgumentException("Only algorithms with a DataAnalysisProblem could be used for the cross validation.");
|
---|
| 127 | if (algorithm != null) DeregisterAlgorithmEvents();
|
---|
| 128 | algorithm = value;
|
---|
| 129 | Parameters.Clear();
|
---|
| 130 |
|
---|
| 131 | if (algorithm != null) {
|
---|
| 132 | algorithm.StoreAlgorithmInEachRun = false;
|
---|
| 133 | RegisterAlgorithmEvents();
|
---|
| 134 | algorithm.Prepare(true);
|
---|
| 135 | Parameters.AddRange(algorithm.Parameters);
|
---|
| 136 | }
|
---|
| 137 | OnAlgorithmChanged();
|
---|
| 138 | Prepare();
|
---|
| 139 | }
|
---|
| 140 | }
|
---|
| 141 | }
|
---|
| 142 |
|
---|
| 143 |
|
---|
| 144 | [Storable]
|
---|
[5659] | 145 | private IDataAnalysisProblem problem;
|
---|
| 146 | public IDataAnalysisProblem Problem {
|
---|
[5617] | 147 | get {
|
---|
| 148 | if (algorithm == null)
|
---|
| 149 | return null;
|
---|
[5659] | 150 | return (IDataAnalysisProblem)algorithm.Problem;
|
---|
[5617] | 151 | }
|
---|
| 152 | set {
|
---|
| 153 | if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
|
---|
| 154 | throw new InvalidOperationException("Changing the problem is only allowed if the CrossValidation is stopped or prepared.");
|
---|
| 155 | if (algorithm == null) throw new ArgumentNullException("Could not set a problem before an algorithm was set.");
|
---|
| 156 | algorithm.Problem = value;
|
---|
| 157 | problem = value;
|
---|
| 158 | }
|
---|
| 159 | }
|
---|
| 160 |
|
---|
| 161 | IProblem IAlgorithm.Problem {
|
---|
| 162 | get { return Problem; }
|
---|
| 163 | set {
|
---|
| 164 | if (value != null && !ProblemType.IsInstanceOfType(value))
|
---|
| 165 | throw new ArgumentException("Only DataAnalysisProblems could be used for the cross validation.");
|
---|
[5659] | 166 | Problem = (IDataAnalysisProblem)value;
|
---|
[5617] | 167 | }
|
---|
| 168 | }
|
---|
| 169 | public Type ProblemType {
|
---|
[5659] | 170 | get { return typeof(IDataAnalysisProblem); }
|
---|
[5617] | 171 | }
|
---|
| 172 |
|
---|
| 173 | [Storable]
|
---|
| 174 | private ItemCollection<IAlgorithm> clonedAlgorithms;
|
---|
| 175 |
|
---|
| 176 | public IEnumerable<IOptimizer> NestedOptimizers {
|
---|
| 177 | get {
|
---|
| 178 | if (Algorithm == null) yield break;
|
---|
| 179 | yield return Algorithm;
|
---|
| 180 | }
|
---|
| 181 | }
|
---|
| 182 |
|
---|
| 183 | [Storable]
|
---|
| 184 | private ResultCollection results;
|
---|
| 185 | public ResultCollection Results {
|
---|
| 186 | get { return results; }
|
---|
| 187 | }
|
---|
| 188 | [Storable]
|
---|
[14864] | 189 | private BoolValue shuffleSamples;
|
---|
| 190 | public BoolValue ShuffleSamples {
|
---|
| 191 | get { return shuffleSamples; }
|
---|
| 192 | }
|
---|
| 193 | [Storable]
|
---|
[5617] | 194 | private IntValue folds;
|
---|
| 195 | public IntValue Folds {
|
---|
| 196 | get { return folds; }
|
---|
| 197 | }
|
---|
| 198 | [Storable]
|
---|
| 199 | private IntValue samplesStart;
|
---|
| 200 | public IntValue SamplesStart {
|
---|
| 201 | get { return samplesStart; }
|
---|
| 202 | }
|
---|
| 203 | [Storable]
|
---|
| 204 | private IntValue samplesEnd;
|
---|
| 205 | public IntValue SamplesEnd {
|
---|
| 206 | get { return samplesEnd; }
|
---|
| 207 | }
|
---|
| 208 | [Storable]
|
---|
| 209 | private IntValue numberOfWorkers;
|
---|
| 210 | public IntValue NumberOfWorkers {
|
---|
| 211 | get { return numberOfWorkers; }
|
---|
| 212 | }
|
---|
| 213 |
|
---|
| 214 | [Storable]
|
---|
| 215 | private bool storeAlgorithmInEachRun;
|
---|
| 216 | public bool StoreAlgorithmInEachRun {
|
---|
| 217 | get { return storeAlgorithmInEachRun; }
|
---|
| 218 | set {
|
---|
| 219 | if (storeAlgorithmInEachRun != value) {
|
---|
| 220 | storeAlgorithmInEachRun = value;
|
---|
| 221 | OnStoreAlgorithmInEachRunChanged();
|
---|
| 222 | }
|
---|
| 223 | }
|
---|
| 224 | }
|
---|
| 225 |
|
---|
| 226 | [Storable]
|
---|
| 227 | private int runsCounter;
|
---|
| 228 | [Storable]
|
---|
| 229 | private RunCollection runs;
|
---|
| 230 | public RunCollection Runs {
|
---|
| 231 | get { return runs; }
|
---|
| 232 | }
|
---|
| 233 |
|
---|
| 234 | [Storable]
|
---|
| 235 | private ExecutionState executionState;
|
---|
| 236 | public ExecutionState ExecutionState {
|
---|
| 237 | get { return executionState; }
|
---|
| 238 | private set {
|
---|
| 239 | if (executionState != value) {
|
---|
| 240 | executionState = value;
|
---|
| 241 | OnExecutionStateChanged();
|
---|
| 242 | OnItemImageChanged();
|
---|
| 243 | }
|
---|
| 244 | }
|
---|
| 245 | }
|
---|
[7201] | 246 | public static new Image StaticItemImage {
|
---|
| 247 | get { return HeuristicLab.Common.Resources.VSImageLibrary.Event; }
|
---|
| 248 | }
|
---|
[5617] | 249 | public override Image ItemImage {
|
---|
| 250 | get {
|
---|
| 251 | if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
|
---|
| 252 | else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
|
---|
| 253 | else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
|
---|
| 254 | else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
|
---|
[7201] | 255 | else return base.ItemImage;
|
---|
[5617] | 256 | }
|
---|
| 257 | }
|
---|
| 258 |
|
---|
| 259 | public TimeSpan ExecutionTime {
|
---|
| 260 | get {
|
---|
| 261 | if (ExecutionState != ExecutionState.Prepared)
|
---|
| 262 | return TimeSpan.FromMilliseconds(clonedAlgorithms.Select(x => x.ExecutionTime.TotalMilliseconds).Sum());
|
---|
| 263 | return TimeSpan.Zero;
|
---|
| 264 | }
|
---|
| 265 | }
|
---|
| 266 | #endregion
|
---|
| 267 |
|
---|
[8738] | 268 | protected override void OnNameChanged() {
|
---|
| 269 | base.OnNameChanged();
|
---|
[8962] | 270 | Runs.OptimizerName = Name;
|
---|
[8738] | 271 | }
|
---|
| 272 |
|
---|
[5617] | 273 | public void Prepare() {
|
---|
[15287] | 274 | if (startPending) return;
|
---|
[5617] | 275 | if (ExecutionState == ExecutionState.Started)
|
---|
| 276 | throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
|
---|
| 277 | results.Clear();
|
---|
| 278 | clonedAlgorithms.Clear();
|
---|
| 279 | if (Algorithm != null) {
|
---|
| 280 | Algorithm.Prepare();
|
---|
| 281 | if (Algorithm.ExecutionState == ExecutionState.Prepared) OnPrepared();
|
---|
| 282 | }
|
---|
| 283 | }
|
---|
| 284 | public void Prepare(bool clearRuns) {
|
---|
| 285 | if (clearRuns) runs.Clear();
|
---|
| 286 | Prepare();
|
---|
| 287 | }
|
---|
| 288 |
|
---|
[15287] | 289 | private bool startPending;
|
---|
[5617] | 290 | public void Start() {
|
---|
[15287] | 291 | Start(CancellationToken.None);
|
---|
| 292 | }
|
---|
| 293 | public void Start(CancellationToken cancellationToken) {
|
---|
| 294 | lock (locker) {
|
---|
| 295 | if (startPending) return;
|
---|
| 296 | startPending = true;
|
---|
| 297 | }
|
---|
[5617] | 298 |
|
---|
[15287] | 299 | try {
|
---|
| 300 | if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
|
---|
| 301 | throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));
|
---|
| 302 | seed = new FastRandom().NextInt();
|
---|
[15002] | 303 |
|
---|
[15287] | 304 | if (Algorithm == null) return;
|
---|
[5617] | 305 | //create cloned algorithms
|
---|
| 306 | if (clonedAlgorithms.Count == 0) {
|
---|
| 307 | int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / Folds.Value;
|
---|
[14864] | 308 | IDataset shuffledDataset = null;
|
---|
[5617] | 309 | for (int i = 0; i < Folds.Value; i++) {
|
---|
[14864] | 310 | var cloner = new Cloner();
|
---|
| 311 | if (ShuffleSamples.Value) {
|
---|
[15002] | 312 | var random = new FastRandom(seed);
|
---|
[14864] | 313 | var dataAnalysisProblem = (IDataAnalysisProblem)algorithm.Problem;
|
---|
| 314 | var dataset = (Dataset)dataAnalysisProblem.ProblemData.Dataset;
|
---|
[15002] | 315 | shuffledDataset = shuffledDataset ?? dataset.Shuffle(random);
|
---|
[14864] | 316 | cloner.RegisterClonedObject(dataset, shuffledDataset);
|
---|
| 317 | }
|
---|
| 318 | IAlgorithm clonedAlgorithm = cloner.Clone(Algorithm);
|
---|
[5617] | 319 | clonedAlgorithm.Name = algorithm.Name + " Fold " + i;
|
---|
[5659] | 320 | IDataAnalysisProblem problem = clonedAlgorithm.Problem as IDataAnalysisProblem;
|
---|
[5886] | 321 | ISymbolicDataAnalysisProblem symbolicProblem = problem as ISymbolicDataAnalysisProblem;
|
---|
| 322 |
|
---|
| 323 | int testStart = (i * testSamplesCount) + SamplesStart.Value;
|
---|
| 324 | int testEnd = (i + 1) == Folds.Value ? SamplesEnd.Value : (i + 1) * testSamplesCount + SamplesStart.Value;
|
---|
| 325 |
|
---|
[7107] | 326 | problem.ProblemData.TrainingPartition.Start = SamplesStart.Value;
|
---|
| 327 | problem.ProblemData.TrainingPartition.End = SamplesEnd.Value;
|
---|
[5886] | 328 | problem.ProblemData.TestPartition.Start = testStart;
|
---|
| 329 | problem.ProblemData.TestPartition.End = testEnd;
|
---|
| 330 | DataAnalysisProblemData problemData = problem.ProblemData as DataAnalysisProblemData;
|
---|
| 331 | if (problemData != null) {
|
---|
| 332 | problemData.TrainingPartitionParameter.Hidden = false;
|
---|
| 333 | problemData.TestPartitionParameter.Hidden = false;
|
---|
| 334 | }
|
---|
| 335 |
|
---|
| 336 | if (symbolicProblem != null) {
|
---|
| 337 | symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
|
---|
| 338 | symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
|
---|
| 339 | }
|
---|
[7738] | 340 | clonedAlgorithm.Prepare();
|
---|
[5617] | 341 | clonedAlgorithms.Add(clonedAlgorithm);
|
---|
| 342 | }
|
---|
| 343 | }
|
---|
| 344 |
|
---|
[15287] | 345 | OnStarted();
|
---|
| 346 | } finally {
|
---|
| 347 | if (startPending) startPending = false;
|
---|
| 348 | }
|
---|
[8836] | 349 |
|
---|
[15287] | 350 | availableWorkers = new SemaphoreSlim(NumberOfWorkers.Value, NumberOfWorkers.Value);
|
---|
| 351 | allAlgorithmsFinished = new ManualResetEventSlim(false);
|
---|
[8836] | 352 |
|
---|
[15367] | 353 | var startedTasks = new List<Task>(clonedAlgorithms.Count);
|
---|
| 354 |
|
---|
[15287] | 355 | //start prepared or paused cloned algorithms
|
---|
| 356 | foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
|
---|
| 357 | if (pausePending || stopPending || ExecutionState != ExecutionState.Started) break;
|
---|
| 358 | if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared ||
|
---|
| 359 | clonedAlgorithm.ExecutionState == ExecutionState.Paused) {
|
---|
| 360 | availableWorkers.Wait();
|
---|
| 361 | lock (locker) {
|
---|
| 362 | if (pausePending || stopPending || ExecutionState != ExecutionState.Started) break;
|
---|
[15367] | 363 | var task = clonedAlgorithm.StartAsync(cancellationToken);
|
---|
| 364 | startedTasks.Add(task);
|
---|
[5617] | 365 | }
|
---|
| 366 | }
|
---|
| 367 | }
|
---|
[15287] | 368 |
|
---|
| 369 | allAlgorithmsFinished.Wait();
|
---|
[15367] | 370 |
|
---|
| 371 | Task.WaitAll(startedTasks.ToArray()); // to get exceptions not handled within the tasks
|
---|
[5617] | 372 | }
|
---|
| 373 |
|
---|
[15287] | 374 | public async Task StartAsync() { await StartAsync(CancellationToken.None); }
|
---|
| 375 | public async Task StartAsync(CancellationToken cancellationToken) {
|
---|
| 376 | await AsyncHelper.DoAsync(Start, cancellationToken);
|
---|
| 377 | }
|
---|
| 378 |
|
---|
[5617] | 379 | private bool pausePending;
|
---|
| 380 | public void Pause() {
|
---|
[15287] | 381 | if (startPending) return;
|
---|
[5617] | 382 | if (ExecutionState != ExecutionState.Started)
|
---|
| 383 | throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
|
---|
| 384 | if (!pausePending) {
|
---|
| 385 | pausePending = true;
|
---|
[15287] | 386 | lock (locker) {
|
---|
| 387 | var toPause = clonedAlgorithms.Where(x => x.ExecutionState == ExecutionState.Started).ToList();
|
---|
| 388 | foreach (var optimizer in toPause) {
|
---|
| 389 | // a race-condition may occur when the optimizer has changed the state by itself in the meantime
|
---|
| 390 | try { optimizer.Pause(); } catch (InvalidOperationException) { }
|
---|
| 391 | }
|
---|
| 392 | }
|
---|
[5617] | 393 | }
|
---|
| 394 | }
|
---|
| 395 |
|
---|
| 396 | private bool stopPending;
|
---|
| 397 | public void Stop() {
|
---|
[15287] | 398 | if (startPending) return;
|
---|
[5617] | 399 | if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
|
---|
| 400 | throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".",
|
---|
| 401 | ExecutionState));
|
---|
| 402 | if (!stopPending) {
|
---|
| 403 | stopPending = true;
|
---|
[15287] | 404 | lock (locker) {
|
---|
| 405 | var toStop = clonedAlgorithms.Where(x => x.ExecutionState == ExecutionState.Started || x.ExecutionState == ExecutionState.Paused).ToList();
|
---|
| 406 | foreach (var optimizer in toStop) {
|
---|
| 407 | // a race-condition may occur when the optimizer has changed the state by itself in the meantime
|
---|
| 408 | try { optimizer.Stop(); } catch (InvalidOperationException) { }
|
---|
| 409 | }
|
---|
| 410 | }
|
---|
[5617] | 411 | }
|
---|
| 412 | }
|
---|
| 413 |
|
---|
| 414 | #region collect parameters and results
|
---|
| 415 | public override void CollectParameterValues(IDictionary<string, IItem> values) {
|
---|
| 416 | values.Add("Algorithm Name", new StringValue(Name));
|
---|
| 417 | values.Add("Algorithm Type", new StringValue(GetType().GetPrettyName()));
|
---|
| 418 | values.Add("Folds", new IntValue(Folds.Value));
|
---|
| 419 |
|
---|
| 420 | if (algorithm != null) {
|
---|
| 421 | values.Add("CrossValidation Algorithm Name", new StringValue(Algorithm.Name));
|
---|
| 422 | values.Add("CrossValidation Algorithm Type", new StringValue(Algorithm.GetType().GetPrettyName()));
|
---|
| 423 | base.CollectParameterValues(values);
|
---|
| 424 | }
|
---|
| 425 | if (Problem != null) {
|
---|
| 426 | values.Add("Problem Name", new StringValue(Problem.Name));
|
---|
| 427 | values.Add("Problem Type", new StringValue(Problem.GetType().GetPrettyName()));
|
---|
| 428 | Problem.CollectParameterValues(values);
|
---|
| 429 | }
|
---|
| 430 | }
|
---|
| 431 |
|
---|
| 432 | public void CollectResultValues(IDictionary<string, IItem> results) {
|
---|
[6566] | 433 | var clonedResults = (ResultCollection)this.results.Clone();
|
---|
| 434 | foreach (var result in clonedResults) {
|
---|
| 435 | results.Add(result.Name, result.Value);
|
---|
| 436 | }
|
---|
| 437 | }
|
---|
| 438 |
|
---|
| 439 | private void AggregateResultValues(IDictionary<string, IItem> results) {
|
---|
[5617] | 440 | IEnumerable<IRun> runs = clonedAlgorithms.Select(alg => alg.Runs.FirstOrDefault()).Where(run => run != null);
|
---|
| 441 | IEnumerable<KeyValuePair<string, IItem>> resultCollections = runs.Where(x => x != null).SelectMany(x => x.Results).ToList();
|
---|
| 442 |
|
---|
| 443 | foreach (IResult result in ExtractAndAggregateResults<IntValue>(resultCollections))
|
---|
| 444 | results.Add(result.Name, result.Value);
|
---|
| 445 | foreach (IResult result in ExtractAndAggregateResults<DoubleValue>(resultCollections))
|
---|
| 446 | results.Add(result.Name, result.Value);
|
---|
| 447 | foreach (IResult result in ExtractAndAggregateResults<PercentValue>(resultCollections))
|
---|
| 448 | results.Add(result.Name, result.Value);
|
---|
[6184] | 449 | foreach (IResult result in ExtractAndAggregateRegressionSolutions(resultCollections)) {
|
---|
| 450 | results.Add(result.Name, result.Value);
|
---|
| 451 | }
|
---|
[6239] | 452 | foreach (IResult result in ExtractAndAggregateClassificationSolutions(resultCollections)) {
|
---|
| 453 | results.Add(result.Name, result.Value);
|
---|
| 454 | }
|
---|
[5617] | 455 | results.Add("Execution Time", new TimeSpanValue(this.ExecutionTime));
|
---|
| 456 | results.Add("CrossValidation Folds", new RunCollection(runs));
|
---|
| 457 | }
|
---|
| 458 |
|
---|
[6184] | 459 | private IEnumerable<IResult> ExtractAndAggregateRegressionSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
|
---|
| 460 | Dictionary<string, List<IRegressionSolution>> resultSolutions = new Dictionary<string, List<IRegressionSolution>>();
|
---|
| 461 | foreach (var result in resultCollections) {
|
---|
| 462 | var regressionSolution = result.Value as IRegressionSolution;
|
---|
| 463 | if (regressionSolution != null) {
|
---|
| 464 | if (resultSolutions.ContainsKey(result.Key)) {
|
---|
| 465 | resultSolutions[result.Key].Add(regressionSolution);
|
---|
| 466 | } else {
|
---|
| 467 | resultSolutions.Add(result.Key, new List<IRegressionSolution>() { regressionSolution });
|
---|
| 468 | }
|
---|
| 469 | }
|
---|
| 470 | }
|
---|
| 471 | List<IResult> aggregatedResults = new List<IResult>();
|
---|
| 472 | foreach (KeyValuePair<string, List<IRegressionSolution>> solutions in resultSolutions) {
|
---|
[6566] | 473 | // clone manually to correctly clone references between cloned root objects
|
---|
| 474 | Cloner cloner = new Cloner();
|
---|
[15002] | 475 | if (ShuffleSamples.Value) {
|
---|
| 476 | var dataset = (Dataset)Problem.ProblemData.Dataset;
|
---|
| 477 | var random = new FastRandom(seed);
|
---|
| 478 | var shuffledDataset = dataset.Shuffle(random);
|
---|
| 479 | cloner.RegisterClonedObject(dataset, shuffledDataset);
|
---|
| 480 | }
|
---|
| 481 | var problemDataClone = (IRegressionProblemData)cloner.Clone(Problem.ProblemData);
|
---|
[6566] | 482 | // set partitions of problem data clone correctly
|
---|
[6184] | 483 | problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
|
---|
| 484 | problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
|
---|
[6566] | 485 | // clone models
|
---|
[7738] | 486 | var ensembleSolution = new RegressionEnsembleSolution(problemDataClone);
|
---|
| 487 | ensembleSolution.AddRegressionSolutions(solutions.Value);
|
---|
[6184] | 488 |
|
---|
[6250] | 489 | aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
|
---|
[6184] | 490 | }
|
---|
[6250] | 491 | List<IResult> flattenedResults = new List<IResult>();
|
---|
| 492 | CollectResultsRecursively("", aggregatedResults, flattenedResults);
|
---|
| 493 | return flattenedResults;
|
---|
[6184] | 494 | }
|
---|
| 495 |
|
---|
[6239] | 496 | private IEnumerable<IResult> ExtractAndAggregateClassificationSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
|
---|
| 497 | Dictionary<string, List<IClassificationSolution>> resultSolutions = new Dictionary<string, List<IClassificationSolution>>();
|
---|
| 498 | foreach (var result in resultCollections) {
|
---|
| 499 | var classificationSolution = result.Value as IClassificationSolution;
|
---|
| 500 | if (classificationSolution != null) {
|
---|
| 501 | if (resultSolutions.ContainsKey(result.Key)) {
|
---|
| 502 | resultSolutions[result.Key].Add(classificationSolution);
|
---|
| 503 | } else {
|
---|
| 504 | resultSolutions.Add(result.Key, new List<IClassificationSolution>() { classificationSolution });
|
---|
| 505 | }
|
---|
| 506 | }
|
---|
| 507 | }
|
---|
[6250] | 508 | var aggregatedResults = new List<IResult>();
|
---|
[6239] | 509 | foreach (KeyValuePair<string, List<IClassificationSolution>> solutions in resultSolutions) {
|
---|
[14781] | 510 | // at least one algorithm (GBT with logistic regression loss) produces a classification solution even though the original problem is a regression problem.
|
---|
| 511 | var targetVariable = solutions.Value.First().ProblemData.TargetVariable;
|
---|
[15002] | 512 | var dataset = (Dataset)Problem.ProblemData.Dataset;
|
---|
| 513 | if (ShuffleSamples.Value) {
|
---|
| 514 | var random = new FastRandom(seed);
|
---|
| 515 | dataset = dataset.Shuffle(random);
|
---|
| 516 | }
|
---|
| 517 | var problemDataClone = new ClassificationProblemData(dataset, Problem.ProblemData.AllowedInputVariables, targetVariable);
|
---|
[6566] | 518 | // set partitions of problem data clone correctly
|
---|
[6239] | 519 | problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
|
---|
| 520 | problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
|
---|
[6566] | 521 | // clone models
|
---|
[8528] | 522 | var ensembleSolution = new ClassificationEnsembleSolution(problemDataClone);
|
---|
| 523 | ensembleSolution.AddClassificationSolutions(solutions.Value);
|
---|
[6239] | 524 |
|
---|
[6250] | 525 | aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
|
---|
[6239] | 526 | }
|
---|
[6250] | 527 | List<IResult> flattenedResults = new List<IResult>();
|
---|
| 528 | CollectResultsRecursively("", aggregatedResults, flattenedResults);
|
---|
| 529 | return flattenedResults;
|
---|
[6239] | 530 | }
|
---|
| 531 |
|
---|
[6250] | 532 | private void CollectResultsRecursively(string path, IEnumerable<IResult> results, IList<IResult> flattenedResults) {
|
---|
| 533 | foreach (IResult result in results) {
|
---|
| 534 | flattenedResults.Add(new Result(path + result.Name, result.Value));
|
---|
| 535 | ResultCollection childCollection = result.Value as ResultCollection;
|
---|
| 536 | if (childCollection != null) {
|
---|
| 537 | CollectResultsRecursively(path + result.Name + ".", childCollection, flattenedResults);
|
---|
| 538 | }
|
---|
| 539 | }
|
---|
| 540 | }
|
---|
| 541 |
|
---|
[5617] | 542 | private static IEnumerable<IResult> ExtractAndAggregateResults<T>(IEnumerable<KeyValuePair<string, IItem>> results)
|
---|
| 543 | where T : class, IItem, new() {
|
---|
| 544 | Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
|
---|
| 545 | foreach (var resultValue in results.Where(r => r.Value.GetType() == typeof(T))) {
|
---|
| 546 | if (!resultValues.ContainsKey(resultValue.Key))
|
---|
| 547 | resultValues[resultValue.Key] = new List<double>();
|
---|
| 548 | resultValues[resultValue.Key].Add(ConvertToDouble(resultValue.Value));
|
---|
| 549 | }
|
---|
| 550 |
|
---|
| 551 | DoubleValue doubleValue;
|
---|
| 552 | if (typeof(T) == typeof(PercentValue))
|
---|
| 553 | doubleValue = new PercentValue();
|
---|
| 554 | else if (typeof(T) == typeof(DoubleValue))
|
---|
| 555 | doubleValue = new DoubleValue();
|
---|
| 556 | else if (typeof(T) == typeof(IntValue))
|
---|
| 557 | doubleValue = new DoubleValue();
|
---|
| 558 | else
|
---|
| 559 | throw new NotSupportedException();
|
---|
| 560 |
|
---|
| 561 | List<IResult> aggregatedResults = new List<IResult>();
|
---|
| 562 | foreach (KeyValuePair<string, List<double>> resultValue in resultValues) {
|
---|
| 563 | doubleValue.Value = resultValue.Value.Average();
|
---|
[6250] | 564 | aggregatedResults.Add(new Result(resultValue.Key + " (average)", (IItem)doubleValue.Clone()));
|
---|
[5617] | 565 | doubleValue.Value = resultValue.Value.StandardDeviation();
|
---|
[6250] | 566 | aggregatedResults.Add(new Result(resultValue.Key + " (std.dev.)", (IItem)doubleValue.Clone()));
|
---|
[5617] | 567 | }
|
---|
| 568 | return aggregatedResults;
|
---|
| 569 | }
|
---|
| 570 |
|
---|
| 571 | private static double ConvertToDouble(IItem item) {
|
---|
| 572 | if (item is DoubleValue) return ((DoubleValue)item).Value;
|
---|
| 573 | else if (item is IntValue) return ((IntValue)item).Value;
|
---|
| 574 | else throw new NotSupportedException("Could not convert any item type to double");
|
---|
| 575 | }
|
---|
| 576 | #endregion
|
---|
| 577 |
|
---|
| 578 | #region events
|
---|
| 579 | private void RegisterEvents() {
|
---|
| 580 | Folds.ValueChanged += new EventHandler(Folds_ValueChanged);
|
---|
| 581 | RegisterClonedAlgorithmsEvents();
|
---|
| 582 | }
|
---|
| 583 | private void Folds_ValueChanged(object sender, EventArgs e) {
|
---|
| 584 | if (ExecutionState != ExecutionState.Prepared)
|
---|
| 585 | throw new InvalidOperationException("Can not change number of folds if the execution state is not prepared.");
|
---|
| 586 | }
|
---|
[7107] | 587 |
|
---|
[5617] | 588 |
|
---|
| 589 | #region template algorithms events
|
---|
| 590 | public event EventHandler AlgorithmChanged;
|
---|
| 591 | private void OnAlgorithmChanged() {
|
---|
| 592 | EventHandler handler = AlgorithmChanged;
|
---|
| 593 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
| 594 | OnProblemChanged();
|
---|
| 595 | if (Problem == null) ExecutionState = ExecutionState.Stopped;
|
---|
| 596 | }
|
---|
| 597 | private void RegisterAlgorithmEvents() {
|
---|
| 598 | algorithm.ProblemChanged += new EventHandler(Algorithm_ProblemChanged);
|
---|
| 599 | algorithm.ExecutionStateChanged += new EventHandler(Algorithm_ExecutionStateChanged);
|
---|
[14904] | 600 | if (Problem != null) {
|
---|
| 601 | Problem.Reset += new EventHandler(Problem_Reset);
|
---|
| 602 | }
|
---|
[5617] | 603 | }
|
---|
| 604 | private void DeregisterAlgorithmEvents() {
|
---|
| 605 | algorithm.ProblemChanged -= new EventHandler(Algorithm_ProblemChanged);
|
---|
| 606 | algorithm.ExecutionStateChanged -= new EventHandler(Algorithm_ExecutionStateChanged);
|
---|
[14904] | 607 | if (Problem != null) {
|
---|
| 608 | Problem.Reset -= new EventHandler(Problem_Reset);
|
---|
| 609 | }
|
---|
[5617] | 610 | }
|
---|
| 611 | private void Algorithm_ProblemChanged(object sender, EventArgs e) {
|
---|
[5659] | 612 | if (algorithm.Problem != null && !(algorithm.Problem is IDataAnalysisProblem)) {
|
---|
[5617] | 613 | algorithm.Problem = problem;
|
---|
| 614 | throw new ArgumentException("A cross validation algorithm can only contain DataAnalysisProblems.");
|
---|
| 615 | }
|
---|
[8970] | 616 | if (problem != null) problem.Reset -= new EventHandler(Problem_Reset);
|
---|
[5659] | 617 | problem = (IDataAnalysisProblem)algorithm.Problem;
|
---|
[9517] | 618 | if (problem != null) problem.Reset += new EventHandler(Problem_Reset);
|
---|
[5617] | 619 | OnProblemChanged();
|
---|
| 620 | }
|
---|
| 621 | public event EventHandler ProblemChanged;
|
---|
| 622 | private void OnProblemChanged() {
|
---|
| 623 | EventHandler handler = ProblemChanged;
|
---|
| 624 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
[8969] | 625 | ConfigureProblem();
|
---|
| 626 | }
|
---|
| 627 | private void Problem_Reset(object sender, EventArgs e) {
|
---|
| 628 | ConfigureProblem();
|
---|
| 629 | }
|
---|
[8970] | 630 | private void ConfigureProblem() {
|
---|
| 631 | SamplesStart.Value = 0;
|
---|
[5806] | 632 | if (Problem != null) {
|
---|
[5617] | 633 | SamplesEnd.Value = Problem.ProblemData.Dataset.Rows;
|
---|
[5886] | 634 |
|
---|
| 635 | DataAnalysisProblemData problemData = Problem.ProblemData as DataAnalysisProblemData;
|
---|
| 636 | if (problemData != null) {
|
---|
| 637 | problemData.TrainingPartitionParameter.Hidden = true;
|
---|
| 638 | problemData.TestPartitionParameter.Hidden = true;
|
---|
| 639 | }
|
---|
| 640 | ISymbolicDataAnalysisProblem symbolicProblem = Problem as ISymbolicDataAnalysisProblem;
|
---|
| 641 | if (symbolicProblem != null) {
|
---|
| 642 | symbolicProblem.FitnessCalculationPartitionParameter.Hidden = true;
|
---|
| 643 | symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
|
---|
| 644 | symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
|
---|
| 645 | symbolicProblem.ValidationPartitionParameter.Hidden = true;
|
---|
| 646 | symbolicProblem.ValidationPartition.Start = 0;
|
---|
| 647 | symbolicProblem.ValidationPartition.End = 0;
|
---|
| 648 | }
|
---|
[5806] | 649 | } else
|
---|
[5617] | 650 | SamplesEnd.Value = 0;
|
---|
| 651 | }
|
---|
| 652 |
|
---|
| 653 | private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
|
---|
| 654 | switch (Algorithm.ExecutionState) {
|
---|
[14864] | 655 | case ExecutionState.Prepared:
|
---|
| 656 | OnPrepared();
|
---|
[5617] | 657 | break;
|
---|
| 658 | case ExecutionState.Started: throw new InvalidOperationException("Algorithm template can not be started.");
|
---|
| 659 | case ExecutionState.Paused: throw new InvalidOperationException("Algorithm template can not be paused.");
|
---|
[14864] | 660 | case ExecutionState.Stopped:
|
---|
| 661 | OnStopped();
|
---|
[5617] | 662 | break;
|
---|
| 663 | }
|
---|
| 664 | }
|
---|
| 665 | #endregion
|
---|
| 666 |
|
---|
| 667 | #region clonedAlgorithms events
|
---|
| 668 | private void RegisterClonedAlgorithmsEvents() {
|
---|
| 669 | clonedAlgorithms.ItemsAdded += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
|
---|
| 670 | clonedAlgorithms.ItemsRemoved += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
|
---|
| 671 | clonedAlgorithms.CollectionReset += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
|
---|
| 672 | foreach (IAlgorithm algorithm in clonedAlgorithms)
|
---|
| 673 | RegisterClonedAlgorithmEvents(algorithm);
|
---|
| 674 | }
|
---|
| 675 | private void DeregisterClonedAlgorithmsEvents() {
|
---|
| 676 | clonedAlgorithms.ItemsAdded -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
|
---|
| 677 | clonedAlgorithms.ItemsRemoved -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
|
---|
| 678 | clonedAlgorithms.CollectionReset -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
|
---|
| 679 | foreach (IAlgorithm algorithm in clonedAlgorithms)
|
---|
| 680 | DeregisterClonedAlgorithmEvents(algorithm);
|
---|
| 681 | }
|
---|
| 682 | private void ClonedAlgorithms_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
|
---|
| 683 | foreach (IAlgorithm algorithm in e.Items)
|
---|
| 684 | RegisterClonedAlgorithmEvents(algorithm);
|
---|
| 685 | }
|
---|
| 686 | private void ClonedAlgorithms_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
|
---|
| 687 | foreach (IAlgorithm algorithm in e.Items)
|
---|
| 688 | DeregisterClonedAlgorithmEvents(algorithm);
|
---|
| 689 | }
|
---|
| 690 | private void ClonedAlgorithms_CollectionReset(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
|
---|
| 691 | foreach (IAlgorithm algorithm in e.OldItems)
|
---|
| 692 | DeregisterClonedAlgorithmEvents(algorithm);
|
---|
| 693 | foreach (IAlgorithm algorithm in e.Items)
|
---|
| 694 | RegisterClonedAlgorithmEvents(algorithm);
|
---|
| 695 | }
|
---|
| 696 | private void RegisterClonedAlgorithmEvents(IAlgorithm algorithm) {
|
---|
| 697 | algorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
|
---|
| 698 | algorithm.ExecutionTimeChanged += new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
|
---|
| 699 | algorithm.Started += new EventHandler(ClonedAlgorithm_Started);
|
---|
| 700 | algorithm.Paused += new EventHandler(ClonedAlgorithm_Paused);
|
---|
| 701 | algorithm.Stopped += new EventHandler(ClonedAlgorithm_Stopped);
|
---|
| 702 | }
|
---|
| 703 | private void DeregisterClonedAlgorithmEvents(IAlgorithm algorithm) {
|
---|
| 704 | algorithm.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
|
---|
| 705 | algorithm.ExecutionTimeChanged -= new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
|
---|
| 706 | algorithm.Started -= new EventHandler(ClonedAlgorithm_Started);
|
---|
| 707 | algorithm.Paused -= new EventHandler(ClonedAlgorithm_Paused);
|
---|
| 708 | algorithm.Stopped -= new EventHandler(ClonedAlgorithm_Stopped);
|
---|
| 709 | }
|
---|
| 710 | private void ClonedAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
|
---|
[15287] | 711 | Pause();
|
---|
[5617] | 712 | OnExceptionOccurred(e.Value);
|
---|
| 713 | }
|
---|
| 714 | private void ClonedAlgorithm_ExecutionTimeChanged(object sender, EventArgs e) {
|
---|
| 715 | OnExecutionTimeChanged();
|
---|
| 716 | }
|
---|
| 717 |
|
---|
| 718 | private readonly object locker = new object();
|
---|
[9504] | 719 | private readonly object resultLocker = new object();
|
---|
[5617] | 720 | private void ClonedAlgorithm_Started(object sender, EventArgs e) {
|
---|
[9493] | 721 | IAlgorithm algorithm = sender as IAlgorithm;
|
---|
[9504] | 722 | lock (resultLocker) {
|
---|
| 723 | if (algorithm != null && !results.ContainsKey(algorithm.Name))
|
---|
| 724 | results.Add(new Result(algorithm.Name, "Contains results for the specific fold.", algorithm.Results));
|
---|
| 725 | }
|
---|
[5617] | 726 | }
|
---|
| 727 |
|
---|
| 728 | private void ClonedAlgorithm_Paused(object sender, EventArgs e) {
|
---|
| 729 | lock (locker) {
|
---|
[15287] | 730 | availableWorkers.Release();
|
---|
| 731 | if (clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started)) {
|
---|
[5617] | 732 | OnPaused();
|
---|
[15287] | 733 | allAlgorithmsFinished.Set();
|
---|
| 734 | }
|
---|
[5617] | 735 | }
|
---|
| 736 | }
|
---|
| 737 |
|
---|
| 738 | private void ClonedAlgorithm_Stopped(object sender, EventArgs e) {
|
---|
| 739 | lock (locker) {
|
---|
[15287] | 740 | // if the algorithm was in paused state, its worker has already been released
|
---|
| 741 | if (availableWorkers.CurrentCount < NumberOfWorkers.Value)
|
---|
| 742 | availableWorkers.Release();
|
---|
| 743 | if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped)) {
|
---|
| 744 | OnStopped();
|
---|
| 745 | allAlgorithmsFinished.Set();
|
---|
| 746 | } else if (stopPending && clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped)) {
|
---|
| 747 | OnStopped();
|
---|
| 748 | allAlgorithmsFinished.Set();
|
---|
[5617] | 749 | }
|
---|
| 750 | }
|
---|
| 751 | }
|
---|
| 752 | #endregion
|
---|
| 753 | #endregion
|
---|
| 754 |
|
---|
| 755 | #region event firing
|
---|
| 756 | public event EventHandler ExecutionStateChanged;
|
---|
| 757 | private void OnExecutionStateChanged() {
|
---|
| 758 | EventHandler handler = ExecutionStateChanged;
|
---|
| 759 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
| 760 | }
|
---|
| 761 | public event EventHandler ExecutionTimeChanged;
|
---|
| 762 | private void OnExecutionTimeChanged() {
|
---|
| 763 | EventHandler handler = ExecutionTimeChanged;
|
---|
| 764 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
| 765 | }
|
---|
| 766 | public event EventHandler Prepared;
|
---|
| 767 | private void OnPrepared() {
|
---|
| 768 | ExecutionState = ExecutionState.Prepared;
|
---|
| 769 | EventHandler handler = Prepared;
|
---|
| 770 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
| 771 | OnExecutionTimeChanged();
|
---|
| 772 | }
|
---|
| 773 | public event EventHandler Started;
|
---|
| 774 | private void OnStarted() {
|
---|
[15287] | 775 | startPending = false;
|
---|
[5617] | 776 | ExecutionState = ExecutionState.Started;
|
---|
| 777 | EventHandler handler = Started;
|
---|
| 778 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
| 779 | }
|
---|
| 780 | public event EventHandler Paused;
|
---|
| 781 | private void OnPaused() {
|
---|
| 782 | pausePending = false;
|
---|
| 783 | ExecutionState = ExecutionState.Paused;
|
---|
| 784 | EventHandler handler = Paused;
|
---|
| 785 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
| 786 | }
|
---|
| 787 | public event EventHandler Stopped;
|
---|
| 788 | private void OnStopped() {
|
---|
| 789 | stopPending = false;
|
---|
| 790 | Dictionary<string, IItem> collectedResults = new Dictionary<string, IItem>();
|
---|
[6566] | 791 | AggregateResultValues(collectedResults);
|
---|
[5617] | 792 | results.AddRange(collectedResults.Select(x => new Result(x.Key, x.Value)).Cast<IResult>().ToArray());
|
---|
[14864] | 793 | clonedAlgorithms.Clear();
|
---|
[5617] | 794 | runsCounter++;
|
---|
| 795 | runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
|
---|
| 796 | ExecutionState = ExecutionState.Stopped;
|
---|
| 797 | EventHandler handler = Stopped;
|
---|
| 798 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
| 799 | }
|
---|
| 800 | public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
|
---|
| 801 | private void OnExceptionOccurred(Exception exception) {
|
---|
| 802 | EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
|
---|
| 803 | if (handler != null) handler(this, new EventArgs<Exception>(exception));
|
---|
| 804 | }
|
---|
| 805 | public event EventHandler StoreAlgorithmInEachRunChanged;
|
---|
| 806 | private void OnStoreAlgorithmInEachRunChanged() {
|
---|
| 807 | EventHandler handler = StoreAlgorithmInEachRunChanged;
|
---|
| 808 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
| 809 | }
|
---|
| 810 | #endregion
|
---|
| 811 | }
|
---|
| 812 | }
|
---|