Changeset 15973 for branches/2522_RefactorPluginInfrastructure/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs
- Timestamp:
- 06/28/18 11:13:37 (6 years ago)
- Location:
- branches/2522_RefactorPluginInfrastructure
- Files:
-
- 4 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/2522_RefactorPluginInfrastructure
- Property svn:ignore
-
old new 24 24 protoc.exe 25 25 obj 26 .vs
-
- Property svn:mergeinfo changed
- Property svn:ignore
-
branches/2522_RefactorPluginInfrastructure/HeuristicLab.Algorithms.DataAnalysis
- Property svn:mergeinfo changed
-
branches/2522_RefactorPluginInfrastructure/HeuristicLab.Algorithms.DataAnalysis/3.4
-
Property
svn:mergeinfo
set to
(toggle deleted branches)
/stable/HeuristicLab.Algorithms.DataAnalysis/3.4 merged eligible /trunk/HeuristicLab.Algorithms.DataAnalysis/3.4 merged eligible /branches/1721-RandomForestPersistence/HeuristicLab.Algorithms.DataAnalysis/3.4 10321-10322 /branches/Async/HeuristicLab.Algorithms.DataAnalysis/3.4 13329-15286 /branches/Benchmarking/sources/HeuristicLab.Algorithms.DataAnalysis/3.4 6917-7005 /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4 9070-13099 /branches/CloningRefactoring/HeuristicLab.Algorithms.DataAnalysis/3.4 4656-4721 /branches/DataAnalysis Refactoring/HeuristicLab.Algorithms.DataAnalysis/3.4 5471-5808 /branches/DataAnalysis SolutionEnsembles/HeuristicLab.Algorithms.DataAnalysis/3.4 5815-6180 /branches/DataAnalysis/HeuristicLab.Algorithms.DataAnalysis/3.4 4458-4459,4462,4464 /branches/DataPreprocessing/HeuristicLab.Algorithms.DataAnalysis/3.4 10085-11101 /branches/GP.Grammar.Editor/HeuristicLab.Algorithms.DataAnalysis/3.4 6284-6795 /branches/GP.Symbols (TimeLag, Diff, Integral)/HeuristicLab.Algorithms.DataAnalysis/3.4 5060 /branches/HeuristicLab.DatasetRefactor/sources/HeuristicLab.Algorithms.DataAnalysis/3.4 11570-12508 /branches/HeuristicLab.Problems.Orienteering/HeuristicLab.Algorithms.DataAnalysis/3.4 11130-12721 /branches/HeuristicLab.RegressionSolutionGradientView/HeuristicLab.Algorithms.DataAnalysis/3.4 13819-14091 /branches/HeuristicLab.TimeSeries/HeuristicLab.Algorithms.DataAnalysis/3.4 8116-8789 /branches/LogResidualEvaluator/HeuristicLab.Algorithms.DataAnalysis/3.4 10202-10483 /branches/NET40/sources/HeuristicLab.Algorithms.DataAnalysis/3.4 5138-5162 /branches/ParallelEngine/HeuristicLab.Algorithms.DataAnalysis/3.4 5175-5192 /branches/ProblemInstancesRegressionAndClassification/HeuristicLab.Algorithms.DataAnalysis/3.4 7773-7810 /branches/QAPAlgorithms/HeuristicLab.Algorithms.DataAnalysis/3.4 6350-6627 /branches/Restructure trunk solution/HeuristicLab.Algorithms.DataAnalysis/3.4 6828 /branches/SpectralKernelForGaussianProcesses/HeuristicLab.Algorithms.DataAnalysis/3.4 10204-10479 /branches/SuccessProgressAnalysis/HeuristicLab.Algorithms.DataAnalysis/3.4 5370-5682 /branches/Trunk/HeuristicLab.Algorithms.DataAnalysis/3.4 6829-6865 /branches/VNS/HeuristicLab.Algorithms.DataAnalysis/3.4 5594-5752 /branches/Weighted TSNE/3.4 15451-15531 /branches/histogram/HeuristicLab.Algorithms.DataAnalysis/3.4 5959-6341 /branches/symbreg-factors-2650/HeuristicLab.Algorithms.DataAnalysis/3.4 14232-14825 /trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4 13402-15674
-
Property
svn:mergeinfo
set to
(toggle deleted branches)
-
branches/2522_RefactorPluginInfrastructure/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs
r13238 r15973 1 1 #region License Information 2 2 /* HeuristicLab 3 * Copyright (C) 2002-201 5Heuristic and Evolutionary Algorithms Laboratory (HEAL)3 * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL) 4 4 * 5 5 * This file is part of HeuristicLab. … … 25 25 using System.Linq; 26 26 using System.Threading; 27 using System.Threading.Tasks; 27 28 using HeuristicLab.Collections; 28 29 using HeuristicLab.Common; … … 33 34 using HeuristicLab.Problems.DataAnalysis; 34 35 using HeuristicLab.Problems.DataAnalysis.Symbolic; 36 using HeuristicLab.Random; 35 37 36 38 namespace HeuristicLab.Algorithms.DataAnalysis { … … 39 41 [StorableClass] 40 42 public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent { 43 [Storable] 44 private int seed; 45 46 private SemaphoreSlim availableWorkers; // limits the number of concurrent algorithm executions 47 private ManualResetEventSlim allAlgorithmsFinished; // this indicates that all started algorithms have been paused or stopped 48 41 49 public CrossValidation() 42 50 : base() { … … 56 64 samplesStart = new IntValue(0); 57 65 samplesEnd = new IntValue(0); 66 shuffleSamples = new BoolValue(false); 58 67 storeAlgorithmInEachRun = false; 59 68 … … 71 80 [StorableHook(HookType.AfterDeserialization)] 72 81 private void AfterDeserialization() { 82 // BackwardsCompatibility3.3 83 #region Backwards compatible code, remove with 3.4 84 if (shuffleSamples == null) shuffleSamples = new BoolValue(false); 85 #endregion 86 73 87 RegisterEvents(); 74 88 if (Algorithm != null) RegisterAlgorithmEvents(); … … 89 103 samplesStart = cloner.Clone(original.samplesStart); 90 104 samplesEnd = cloner.Clone(original.samplesEnd); 105 shuffleSamples = cloner.Clone(original.shuffleSamples); 106 seed = original.seed; 107 91 108 RegisterEvents(); 92 109 if (Algorithm != null) RegisterAlgorithmEvents(); … … 170 187 get { return results; } 171 188 } 172 189 [Storable] 190 private BoolValue shuffleSamples; 191 public BoolValue ShuffleSamples { 192 get { return shuffleSamples; } 193 } 173 194 [Storable] 174 195 private IntValue folds; … … 252 273 253 274 public void Prepare() { 275 if (startPending) return; 254 276 if (ExecutionState == ExecutionState.Started) 255 277 throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState)); … … 266 288 } 267 289 290 private bool startPending; 268 291 public void Start() { 269 if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused)) 270 throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState)); 271 272 if (Algorithm != null) { 292 Start(CancellationToken.None); 293 } 294 public void Start(CancellationToken cancellationToken) { 295 lock (locker) { 296 if (startPending) return; 297 startPending = true; 298 } 299 300 try { 301 if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused)) 302 throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState)); 303 seed = new FastRandom().NextInt(); 304 305 if (Algorithm == null) return; 273 306 //create cloned algorithms 274 307 if (clonedAlgorithms.Count == 0) { 275 308 int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / Folds.Value; 276 309 IDataset shuffledDataset = null; 277 310 for (int i = 0; i < Folds.Value; i++) { 278 IAlgorithm clonedAlgorithm = (IAlgorithm)algorithm.Clone(); 311 var cloner = new Cloner(); 312 if (ShuffleSamples.Value) { 313 var random = new FastRandom(seed); 314 var dataAnalysisProblem = (IDataAnalysisProblem)algorithm.Problem; 315 var dataset = (Dataset)dataAnalysisProblem.ProblemData.Dataset; 316 shuffledDataset = shuffledDataset ?? dataset.Shuffle(random); 317 cloner.RegisterClonedObject(dataset, shuffledDataset); 318 } 319 IAlgorithm clonedAlgorithm = cloner.Clone(Algorithm); 279 320 clonedAlgorithm.Name = algorithm.Name + " Fold " + i; 280 321 IDataAnalysisProblem problem = clonedAlgorithm.Problem as IDataAnalysisProblem; … … 303 344 } 304 345 305 //start prepared or paused cloned algorithms 306 int startedAlgorithms = 0; 307 foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) { 308 if (startedAlgorithms < NumberOfWorkers.Value) { 309 if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared || 310 clonedAlgorithm.ExecutionState == ExecutionState.Paused) { 311 312 // start and wait until the alg is started 313 using (var signal = new ManualResetEvent(false)) { 314 EventHandler signalSetter = (sender, args) => { signal.Set(); }; 315 clonedAlgorithm.Started += signalSetter; 316 clonedAlgorithm.Start(); 317 signal.WaitOne(); 318 clonedAlgorithm.Started -= signalSetter; 319 320 startedAlgorithms++; 321 } 322 } 346 OnStarted(); 347 } finally { 348 if (startPending) startPending = false; 349 } 350 351 availableWorkers = new SemaphoreSlim(NumberOfWorkers.Value, NumberOfWorkers.Value); 352 allAlgorithmsFinished = new ManualResetEventSlim(false); 353 354 var startedTasks = new List<Task>(clonedAlgorithms.Count); 355 356 //start prepared or paused cloned algorithms 357 foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) { 358 if (pausePending || stopPending || ExecutionState != ExecutionState.Started) break; 359 if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared || 360 clonedAlgorithm.ExecutionState == ExecutionState.Paused) { 361 availableWorkers.Wait(); 362 lock (locker) { 363 if (pausePending || stopPending || ExecutionState != ExecutionState.Started) break; 364 var task = clonedAlgorithm.StartAsync(cancellationToken); 365 startedTasks.Add(task); 323 366 } 324 367 } 325 OnStarted(); 326 } 368 } 369 370 allAlgorithmsFinished.Wait(); 371 372 Task.WaitAll(startedTasks.ToArray()); // to get exceptions not handled within the tasks 373 } 374 375 public async Task StartAsync() { await StartAsync(CancellationToken.None); } 376 public async Task StartAsync(CancellationToken cancellationToken) { 377 await AsyncHelper.DoAsync(Start, cancellationToken); 327 378 } 328 379 329 380 private bool pausePending; 330 381 public void Pause() { 382 if (startPending) return; 331 383 if (ExecutionState != ExecutionState.Started) 332 384 throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState)); 333 385 if (!pausePending) { 334 386 pausePending = true; 335 PauseAllClonedAlgorithms();336 }337 }338 private void PauseAllClonedAlgorithms() {339 foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {340 if (clonedAlgorithm.ExecutionState == ExecutionState.Started)341 clonedAlgorithm.Pause();387 lock (locker) { 388 var toPause = clonedAlgorithms.Where(x => x.ExecutionState == ExecutionState.Started).ToList(); 389 foreach (var optimizer in toPause) { 390 // a race-condition may occur when the optimizer has changed the state by itself in the meantime 391 try { optimizer.Pause(); } catch (InvalidOperationException) { } 392 } 393 } 342 394 } 343 395 } … … 345 397 private bool stopPending; 346 398 public void Stop() { 399 if (startPending) return; 347 400 if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused)) 348 401 throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".", … … 350 403 if (!stopPending) { 351 404 stopPending = true; 352 StopAllClonedAlgorithms(); 353 } 354 } 355 private void StopAllClonedAlgorithms() { 356 foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) { 357 if (clonedAlgorithm.ExecutionState == ExecutionState.Started || 358 clonedAlgorithm.ExecutionState == ExecutionState.Paused) 359 clonedAlgorithm.Stop(); 405 lock (locker) { 406 var toStop = clonedAlgorithms.Where(x => x.ExecutionState == ExecutionState.Started || x.ExecutionState == ExecutionState.Paused).ToList(); 407 foreach (var optimizer in toStop) { 408 // a race-condition may occur when the optimizer has changed the state by itself in the meantime 409 try { optimizer.Stop(); } catch (InvalidOperationException) { } 410 } 411 } 360 412 } 361 413 } … … 422 474 // clone manually to correctly clone references between cloned root objects 423 475 Cloner cloner = new Cloner(); 476 if (ShuffleSamples.Value) { 477 var dataset = (Dataset)Problem.ProblemData.Dataset; 478 var random = new FastRandom(seed); 479 var shuffledDataset = dataset.Shuffle(random); 480 cloner.RegisterClonedObject(dataset, shuffledDataset); 481 } 424 482 var problemDataClone = (IRegressionProblemData)cloner.Clone(Problem.ProblemData); 425 483 // set partitions of problem data clone correctly … … 451 509 var aggregatedResults = new List<IResult>(); 452 510 foreach (KeyValuePair<string, List<IClassificationSolution>> solutions in resultSolutions) { 453 // clone manually to correctly clone references between cloned root objects 454 Cloner cloner = new Cloner(); 455 var problemDataClone = (IClassificationProblemData)cloner.Clone(Problem.ProblemData); 511 // at least one algorithm (GBT with logistic regression loss) produces a classification solution even though the original problem is a regression problem. 512 var targetVariable = solutions.Value.First().ProblemData.TargetVariable; 513 var dataset = (Dataset)Problem.ProblemData.Dataset; 514 if (ShuffleSamples.Value) { 515 var random = new FastRandom(seed); 516 dataset = dataset.Shuffle(random); 517 } 518 var problemDataClone = new ClassificationProblemData(dataset, Problem.ProblemData.AllowedInputVariables, targetVariable); 456 519 // set partitions of problem data clone correctly 457 520 problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value; … … 536 599 algorithm.ProblemChanged += new EventHandler(Algorithm_ProblemChanged); 537 600 algorithm.ExecutionStateChanged += new EventHandler(Algorithm_ExecutionStateChanged); 538 if (Problem != null) Problem.Reset += new EventHandler(Problem_Reset); 601 if (Problem != null) { 602 Problem.Reset += new EventHandler(Problem_Reset); 603 } 539 604 } 540 605 private void DeregisterAlgorithmEvents() { 541 606 algorithm.ProblemChanged -= new EventHandler(Algorithm_ProblemChanged); 542 607 algorithm.ExecutionStateChanged -= new EventHandler(Algorithm_ExecutionStateChanged); 543 if (Problem != null) Problem.Reset -= new EventHandler(Problem_Reset); 608 if (Problem != null) { 609 Problem.Reset -= new EventHandler(Problem_Reset); 610 } 544 611 } 545 612 private void Algorithm_ProblemChanged(object sender, EventArgs e) { … … 559 626 ConfigureProblem(); 560 627 } 561 562 628 private void Problem_Reset(object sender, EventArgs e) { 563 629 ConfigureProblem(); 564 630 } 565 566 631 private void ConfigureProblem() { 567 632 SamplesStart.Value = 0; … … 589 654 private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) { 590 655 switch (Algorithm.ExecutionState) { 591 case ExecutionState.Prepared: OnPrepared(); 656 case ExecutionState.Prepared: 657 OnPrepared(); 592 658 break; 593 659 case ExecutionState.Started: throw new InvalidOperationException("Algorithm template can not be started."); 594 660 case ExecutionState.Paused: throw new InvalidOperationException("Algorithm template can not be paused."); 595 case ExecutionState.Stopped: OnStopped(); 661 case ExecutionState.Stopped: 662 OnStopped(); 596 663 break; 597 664 } … … 643 710 } 644 711 private void ClonedAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) { 712 Pause(); 645 713 OnExceptionOccurred(e.Value); 646 714 } … … 661 729 private void ClonedAlgorithm_Paused(object sender, EventArgs e) { 662 730 lock (locker) { 663 if (pausePending && clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started)) 731 availableWorkers.Release(); 732 if (clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started)) { 664 733 OnPaused(); 734 allAlgorithmsFinished.Set(); 735 } 665 736 } 666 737 } … … 668 739 private void ClonedAlgorithm_Stopped(object sender, EventArgs e) { 669 740 lock (locker) { 670 if (!stopPending && ExecutionState == ExecutionState.Started) { 671 IAlgorithm preparedAlgorithm = clonedAlgorithms.FirstOrDefault(alg => alg.ExecutionState == ExecutionState.Prepared || 672 alg.ExecutionState == ExecutionState.Paused); 673 if (preparedAlgorithm != null) preparedAlgorithm.Start(); 674 } 675 if (ExecutionState != ExecutionState.Stopped) { 676 if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped)) 677 OnStopped(); 678 else if (stopPending && 679 clonedAlgorithms.All( 680 alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped)) 681 OnStopped(); 741 // if the algorithm was in paused state, its worker has already been released 742 if (availableWorkers.CurrentCount < NumberOfWorkers.Value) 743 availableWorkers.Release(); 744 if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped)) { 745 OnStopped(); 746 allAlgorithmsFinished.Set(); 747 } else if (stopPending && clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped)) { 748 OnStopped(); 749 allAlgorithmsFinished.Set(); 682 750 } 683 751 } … … 706 774 public event EventHandler Started; 707 775 private void OnStarted() { 776 startPending = false; 708 777 ExecutionState = ExecutionState.Started; 709 778 EventHandler handler = Started; … … 723 792 AggregateResultValues(collectedResults); 724 793 results.AddRange(collectedResults.Select(x => new Result(x.Key, x.Value)).Cast<IResult>().ToArray()); 794 clonedAlgorithms.Clear(); 725 795 runsCounter++; 726 796 runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
Note: See TracChangeset
for help on using the changeset viewer.