[4472] | 1 | #region License Information
|
---|
| 2 | /* HeuristicLab
|
---|
| 3 | * Copyright (C) 2002-2010 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
|
---|
| 4 | *
|
---|
| 5 | * This file is part of HeuristicLab.
|
---|
| 6 | *
|
---|
| 7 | * HeuristicLab is free software: you can redistribute it and/or modify
|
---|
| 8 | * it under the terms of the GNU General Public License as published by
|
---|
| 9 | * the Free Software Foundation, either version 3 of the License, or
|
---|
| 10 | * (at your option) any later version.
|
---|
| 11 | *
|
---|
| 12 | * HeuristicLab is distributed in the hope that it will be useful,
|
---|
| 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
---|
| 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
---|
| 15 | * GNU General Public License for more details.
|
---|
| 16 | *
|
---|
| 17 | * You should have received a copy of the GNU General Public License
|
---|
| 18 | * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
|
---|
| 19 | */
|
---|
| 20 | #endregion
|
---|
| 21 |
|
---|
| 22 | using System;
|
---|
[4561] | 23 | using System.Collections.Generic;
|
---|
[4536] | 24 | using System.Drawing;
|
---|
| 25 | using System.Linq;
|
---|
[4472] | 26 | using HeuristicLab.Collections;
|
---|
| 27 | using HeuristicLab.Common;
|
---|
| 28 | using HeuristicLab.Core;
|
---|
| 29 | using HeuristicLab.Data;
|
---|
| 30 | using HeuristicLab.Optimization;
|
---|
| 31 | using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
|
---|
| 32 | using HeuristicLab.Problems.DataAnalysis;
|
---|
| 33 |
|
---|
| 34 | namespace HeuristicLab.Algorithms.DataAnalysis {
|
---|
| 35 | [Item("Cross Validation", "Cross Validation wrapper for data analysis algorithms.")]
|
---|
| 36 | [Creatable("Data Analysis")]
|
---|
| 37 | [StorableClass]
|
---|
[4617] | 38 | public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent {
|
---|
[4472] | 39 | public CrossValidation()
|
---|
| 40 | : base() {
|
---|
| 41 | name = ItemName;
|
---|
| 42 | description = ItemDescription;
|
---|
| 43 |
|
---|
| 44 | executionState = ExecutionState.Stopped;
|
---|
[4542] | 45 | runs = new RunCollection();
|
---|
[4561] | 46 | runsCounter = 0;
|
---|
[4472] | 47 |
|
---|
[4536] | 48 | algorithm = null;
|
---|
[4472] | 49 | clonedAlgorithms = new ItemCollection<IAlgorithm>();
|
---|
| 50 | readOnlyClonedAlgorithms = null;
|
---|
[4561] | 51 | results = new ResultCollection();
|
---|
[4472] | 52 |
|
---|
[4561] | 53 | folds = new IntValue(2);
|
---|
[4542] | 54 | numberOfWorkers = new IntValue(1);
|
---|
[4472] | 55 | samplesStart = new IntValue(0);
|
---|
| 56 | samplesEnd = new IntValue(0);
|
---|
[4561] | 57 | storeAlgorithmInEachRun = false;
|
---|
[4542] | 58 |
|
---|
| 59 | RegisterEvents();
|
---|
[4472] | 60 | }
|
---|
| 61 |
|
---|
[4617] | 62 | public string Filename { get; set; }
|
---|
| 63 |
|
---|
[4542] | 64 | #region persistence and cloning
|
---|
| 65 | [StorableConstructor]
|
---|
| 66 | private CrossValidation(bool deserializing)
|
---|
| 67 | : base(deserializing) {
|
---|
| 68 | }
|
---|
| 69 | [StorableHook(HookType.AfterDeserialization)]
|
---|
| 70 | private void AfterDeserialization() {
|
---|
| 71 | RegisterEvents();
|
---|
| 72 | }
|
---|
| 73 |
|
---|
[4722] | 74 | private CrossValidation(CrossValidation original, Cloner cloner)
|
---|
| 75 | : base(original, cloner) {
|
---|
| 76 | executionState = original.executionState;
|
---|
| 77 | storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
|
---|
| 78 | runs = cloner.Clone(original.runs);
|
---|
| 79 | runsCounter = original.runsCounter;
|
---|
| 80 | algorithm = cloner.Clone(original.algorithm);
|
---|
| 81 | clonedAlgorithms = cloner.Clone(original.clonedAlgorithms);
|
---|
[4735] | 82 | results = cloner.Clone(original.results);
|
---|
| 83 |
|
---|
[4722] | 84 | folds = cloner.Clone(original.folds);
|
---|
| 85 | numberOfWorkers = cloner.Clone(original.numberOfWorkers);
|
---|
| 86 | samplesStart = cloner.Clone(original.samplesStart);
|
---|
| 87 | samplesEnd = cloner.Clone(original.samplesEnd);
|
---|
| 88 | RegisterEvents();
|
---|
| 89 | }
|
---|
[4542] | 90 | public override IDeepCloneable Clone(Cloner cloner) {
|
---|
[4722] | 91 | return new CrossValidation(this, cloner);
|
---|
[4542] | 92 | }
|
---|
[4722] | 93 |
|
---|
[4542] | 94 | #endregion
|
---|
| 95 |
|
---|
[4472] | 96 | #region properties
|
---|
[4542] | 97 | [Storable]
|
---|
[4536] | 98 | private IAlgorithm algorithm;
|
---|
| 99 | public IAlgorithm Algorithm {
|
---|
| 100 | get { return algorithm; }
|
---|
[4472] | 101 | set {
|
---|
[4542] | 102 | if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
|
---|
| 103 | throw new InvalidOperationException("Changing the algorithm is only allowed if the CrossValidation is stopped or prepared.");
|
---|
[4536] | 104 | if (algorithm != value) {
|
---|
[4472] | 105 | if (value != null && value.Problem != null && !(value.Problem is IDataAnalysisProblem))
|
---|
| 106 | throw new ArgumentException("Only algorithms with a DataAnalysisProblem could be used for the cross validation.");
|
---|
[4536] | 107 | if (algorithm != null) DeregisterAlgorithmEvents();
|
---|
| 108 | algorithm = value;
|
---|
[4561] | 109 | Parameters.Clear();
|
---|
[4542] | 110 |
|
---|
| 111 | if (algorithm != null) {
|
---|
[4563] | 112 | algorithm.StoreAlgorithmInEachRun = false;
|
---|
[4542] | 113 | RegisterAlgorithmEvents();
|
---|
| 114 | algorithm.Prepare(true);
|
---|
[4561] | 115 | Parameters.AddRange(algorithm.Parameters);
|
---|
[4542] | 116 | }
|
---|
[4536] | 117 | OnAlgorithmChanged();
|
---|
[4561] | 118 | if (algorithm != null) OnProblemChanged();
|
---|
[4542] | 119 | Prepare();
|
---|
[4472] | 120 | }
|
---|
| 121 | }
|
---|
| 122 | }
|
---|
[4542] | 123 |
|
---|
[4561] | 124 |
|
---|
[4542] | 125 | [Storable]
|
---|
[4563] | 126 | private ISingleObjectiveDataAnalysisProblem problem;
|
---|
| 127 | public ISingleObjectiveDataAnalysisProblem Problem {
|
---|
[4472] | 128 | get {
|
---|
[4536] | 129 | if (algorithm == null)
|
---|
[4472] | 130 | return null;
|
---|
[4563] | 131 | return (ISingleObjectiveDataAnalysisProblem)algorithm.Problem;
|
---|
[4472] | 132 | }
|
---|
[4536] | 133 | set {
|
---|
[4542] | 134 | if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
|
---|
| 135 | throw new InvalidOperationException("Changing the problem is only allowed if the CrossValidation is stopped or prepared.");
|
---|
[4536] | 136 | if (algorithm == null) throw new ArgumentNullException("Could not set a problem before an algorithm was set.");
|
---|
| 137 | algorithm.Problem = value;
|
---|
[4563] | 138 | problem = value;
|
---|
[4536] | 139 | }
|
---|
[4472] | 140 | }
|
---|
[4536] | 141 |
|
---|
[4561] | 142 | IProblem IAlgorithm.Problem {
|
---|
| 143 | get { return Problem; }
|
---|
| 144 | set {
|
---|
| 145 | if (value != null && !ProblemType.IsInstanceOfType(value))
|
---|
| 146 | throw new ArgumentException("Only DataAnalysisProblems could be used for the cross validation.");
|
---|
[4563] | 147 | Problem = (ISingleObjectiveDataAnalysisProblem)value;
|
---|
[4561] | 148 | }
|
---|
| 149 | }
|
---|
| 150 | public Type ProblemType {
|
---|
[4563] | 151 | get { return typeof(ISingleObjectiveDataAnalysisProblem); }
|
---|
[4561] | 152 | }
|
---|
| 153 |
|
---|
[4542] | 154 | [Storable]
|
---|
[4472] | 155 | private ItemCollection<IAlgorithm> clonedAlgorithms;
|
---|
| 156 | private ReadOnlyItemCollection<IAlgorithm> readOnlyClonedAlgorithms;
|
---|
| 157 | public IItemCollection<IAlgorithm> ClonedAlgorithms {
|
---|
| 158 | get {
|
---|
| 159 | if (readOnlyClonedAlgorithms == null) readOnlyClonedAlgorithms = clonedAlgorithms.AsReadOnly();
|
---|
| 160 | return readOnlyClonedAlgorithms;
|
---|
| 161 | }
|
---|
| 162 | }
|
---|
| 163 |
|
---|
[4542] | 164 | [Storable]
|
---|
[4561] | 165 | private ResultCollection results;
|
---|
| 166 | public ResultCollection Results {
|
---|
| 167 | get { return results; }
|
---|
| 168 | }
|
---|
| 169 |
|
---|
| 170 | [Storable]
|
---|
[4472] | 171 | private IntValue folds;
|
---|
| 172 | public IntValue Folds {
|
---|
| 173 | get { return folds; }
|
---|
| 174 | }
|
---|
[4542] | 175 | [Storable]
|
---|
[4472] | 176 | private IntValue samplesStart;
|
---|
| 177 | public IntValue SamplesStart {
|
---|
[4536] | 178 | get { return samplesStart; }
|
---|
[4472] | 179 | }
|
---|
[4542] | 180 | [Storable]
|
---|
[4472] | 181 | private IntValue samplesEnd;
|
---|
| 182 | public IntValue SamplesEnd {
|
---|
[4536] | 183 | get { return samplesEnd; }
|
---|
[4472] | 184 | }
|
---|
[4542] | 185 | [Storable]
|
---|
[4472] | 186 | private IntValue numberOfWorkers;
|
---|
| 187 | public IntValue NumberOfWorkers {
|
---|
| 188 | get { return numberOfWorkers; }
|
---|
| 189 | }
|
---|
| 190 |
|
---|
[4542] | 191 | [Storable]
|
---|
[4561] | 192 | private bool storeAlgorithmInEachRun;
|
---|
| 193 | public bool StoreAlgorithmInEachRun {
|
---|
| 194 | get { return storeAlgorithmInEachRun; }
|
---|
| 195 | set {
|
---|
| 196 | if (storeAlgorithmInEachRun != value) {
|
---|
| 197 | storeAlgorithmInEachRun = value;
|
---|
| 198 | OnStoreAlgorithmInEachRunChanged();
|
---|
| 199 | }
|
---|
| 200 | }
|
---|
| 201 | }
|
---|
| 202 |
|
---|
| 203 | [Storable]
|
---|
| 204 | private int runsCounter;
|
---|
| 205 | [Storable]
|
---|
[4472] | 206 | private RunCollection runs;
|
---|
| 207 | public RunCollection Runs {
|
---|
| 208 | get { return runs; }
|
---|
| 209 | }
|
---|
[4561] | 210 |
|
---|
[4542] | 211 | [Storable]
|
---|
[4472] | 212 | private ExecutionState executionState;
|
---|
| 213 | public ExecutionState ExecutionState {
|
---|
| 214 | get { return executionState; }
|
---|
| 215 | private set {
|
---|
| 216 | if (executionState != value) {
|
---|
| 217 | executionState = value;
|
---|
| 218 | OnExecutionStateChanged();
|
---|
| 219 | OnItemImageChanged();
|
---|
| 220 | }
|
---|
| 221 | }
|
---|
| 222 | }
|
---|
[4536] | 223 | public override Image ItemImage {
|
---|
| 224 | get {
|
---|
| 225 | if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VS2008ImageLibrary.ExecutablePrepared;
|
---|
| 226 | else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VS2008ImageLibrary.ExecutableStarted;
|
---|
| 227 | else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VS2008ImageLibrary.ExecutablePaused;
|
---|
| 228 | else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VS2008ImageLibrary.ExecutableStopped;
|
---|
| 229 | else return HeuristicLab.Common.Resources.VS2008ImageLibrary.Event;
|
---|
| 230 | }
|
---|
| 231 | }
|
---|
| 232 |
|
---|
[4472] | 233 | public TimeSpan ExecutionTime {
|
---|
[4536] | 234 | get {
|
---|
[4567] | 235 | if (ExecutionState != ExecutionState.Prepared)
|
---|
| 236 | return TimeSpan.FromMilliseconds(clonedAlgorithms.Select(x => x.ExecutionTime.TotalMilliseconds).Sum());
|
---|
| 237 | return TimeSpan.Zero;
|
---|
[4536] | 238 | }
|
---|
[4472] | 239 | }
|
---|
| 240 | #endregion
|
---|
| 241 |
|
---|
| 242 | public void Prepare() {
|
---|
[4563] | 243 | if (ExecutionState == ExecutionState.Started)
|
---|
[4542] | 244 | throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
|
---|
[4567] | 245 | results.Clear();
|
---|
[4536] | 246 | clonedAlgorithms.Clear();
|
---|
[4542] | 247 | if (Algorithm != null) {
|
---|
| 248 | Algorithm.Prepare();
|
---|
| 249 | if (Algorithm.ExecutionState == ExecutionState.Prepared) OnPrepared();
|
---|
| 250 | }
|
---|
[4472] | 251 | }
|
---|
| 252 | public void Prepare(bool clearRuns) {
|
---|
| 253 | if (clearRuns) runs.Clear();
|
---|
| 254 | Prepare();
|
---|
| 255 | }
|
---|
[4542] | 256 |
|
---|
| 257 | private bool startPending;
|
---|
[4472] | 258 | public void Start() {
|
---|
[4542] | 259 | if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
|
---|
| 260 | throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));
|
---|
| 261 |
|
---|
| 262 | if (Algorithm != null && !startPending) {
|
---|
| 263 | startPending = true;
|
---|
| 264 | //create cloned algorithms
|
---|
| 265 | if (clonedAlgorithms.Count == 0) {
|
---|
[4561] | 266 | int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / Folds.Value;
|
---|
[4542] | 267 | for (int i = 0; i < Folds.Value; i++) {
|
---|
| 268 | IAlgorithm clonedAlgorithm = (IAlgorithm)algorithm.Clone();
|
---|
| 269 | clonedAlgorithm.Name = algorithm.Name + " Fold " + i;
|
---|
[4561] | 270 | IDataAnalysisProblem problem = clonedAlgorithm.Problem as IDataAnalysisProblem;
|
---|
| 271 | problem.DataAnalysisProblemData.TestSamplesEnd.Value = (i + 1) == Folds.Value ? SamplesEnd.Value : (i + 1) * testSamplesCount + SamplesStart.Value;
|
---|
| 272 | problem.DataAnalysisProblemData.TestSamplesStart.Value = (i * testSamplesCount) + SamplesStart.Value;
|
---|
[4542] | 273 | clonedAlgorithms.Add(clonedAlgorithm);
|
---|
| 274 | }
|
---|
| 275 | }
|
---|
| 276 |
|
---|
| 277 | //start prepared or paused cloned algorithms
|
---|
| 278 | int startedAlgorithms = 0;
|
---|
| 279 | foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
|
---|
| 280 | if (startedAlgorithms < NumberOfWorkers.Value) {
|
---|
| 281 | if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared ||
|
---|
| 282 | clonedAlgorithm.ExecutionState == ExecutionState.Paused) {
|
---|
| 283 | clonedAlgorithm.Start();
|
---|
| 284 | startedAlgorithms++;
|
---|
| 285 | }
|
---|
| 286 | }
|
---|
| 287 | }
|
---|
| 288 | OnStarted();
|
---|
[4536] | 289 | }
|
---|
[4472] | 290 | }
|
---|
[4542] | 291 |
|
---|
| 292 | private bool pausePending;
|
---|
[4472] | 293 | public void Pause() {
|
---|
[4542] | 294 | if (ExecutionState != ExecutionState.Started)
|
---|
| 295 | throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
|
---|
| 296 | if (!pausePending) {
|
---|
| 297 | pausePending = true;
|
---|
| 298 | if (!startPending) PauseAllClonedAlgorithms();
|
---|
| 299 | }
|
---|
[4472] | 300 | }
|
---|
[4542] | 301 | private void PauseAllClonedAlgorithms() {
|
---|
| 302 | foreach (IAlgorithm clonedAlgorithm in ClonedAlgorithms) {
|
---|
| 303 | if (clonedAlgorithm.ExecutionState == ExecutionState.Started)
|
---|
| 304 | clonedAlgorithm.Pause();
|
---|
| 305 | }
|
---|
| 306 | }
|
---|
| 307 |
|
---|
| 308 | private bool stopPending;
|
---|
[4472] | 309 | public void Stop() {
|
---|
[4542] | 310 | if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
|
---|
| 311 | throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".",
|
---|
| 312 | ExecutionState));
|
---|
| 313 | if (!stopPending) {
|
---|
| 314 | stopPending = true;
|
---|
| 315 | if (!startPending) StopAllClonedAlgorithms();
|
---|
| 316 | }
|
---|
[4472] | 317 | }
|
---|
[4542] | 318 | private void StopAllClonedAlgorithms() {
|
---|
| 319 | foreach (IAlgorithm clonedAlgorithm in ClonedAlgorithms) {
|
---|
| 320 | if (clonedAlgorithm.ExecutionState == ExecutionState.Started ||
|
---|
| 321 | clonedAlgorithm.ExecutionState == ExecutionState.Paused)
|
---|
| 322 | clonedAlgorithm.Stop();
|
---|
| 323 | }
|
---|
| 324 | }
|
---|
[4472] | 325 |
|
---|
[4561] | 326 | #region collect parameters and results
|
---|
| 327 | public override void CollectParameterValues(IDictionary<string, IItem> values) {
|
---|
| 328 | values.Add("Algorithm Name", new StringValue(Name));
|
---|
| 329 | values.Add("Algorithm Type", new StringValue(GetType().GetPrettyName()));
|
---|
| 330 | values.Add("Folds", new IntValue(Folds.Value));
|
---|
| 331 |
|
---|
| 332 | if (algorithm != null) {
|
---|
| 333 | values.Add("CrossValidation Algorithm Name", new StringValue(Algorithm.Name));
|
---|
| 334 | values.Add("CrossValidation Algorithm Type", new StringValue(Algorithm.GetType().GetPrettyName()));
|
---|
| 335 | base.CollectParameterValues(values);
|
---|
| 336 | }
|
---|
| 337 | if (Problem != null) {
|
---|
| 338 | values.Add("Problem Name", new StringValue(Problem.Name));
|
---|
| 339 | values.Add("Problem Type", new StringValue(Problem.GetType().GetPrettyName()));
|
---|
| 340 | Problem.CollectParameterValues(values);
|
---|
| 341 | }
|
---|
| 342 | }
|
---|
| 343 |
|
---|
| 344 | public void CollectResultValues(IDictionary<string, IItem> results) {
|
---|
| 345 | Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
|
---|
| 346 | IEnumerable<IRun> runs = ClonedAlgorithms.Select(alg => alg.Runs.FirstOrDefault()).Where(run => run != null);
|
---|
| 347 | IEnumerable<KeyValuePair<string, IItem>> resultCollections = runs.Where(x => x != null).SelectMany(x => x.Results).ToList();
|
---|
| 348 |
|
---|
| 349 | foreach (IResult result in ExtractAndAggregateResults<IntValue>(resultCollections))
|
---|
| 350 | results.Add(result.Name, result.Value);
|
---|
| 351 | foreach (IResult result in ExtractAndAggregateResults<DoubleValue>(resultCollections))
|
---|
| 352 | results.Add(result.Name, result.Value);
|
---|
| 353 | foreach (IResult result in ExtractAndAggregateResults<PercentValue>(resultCollections))
|
---|
| 354 | results.Add(result.Name, result.Value);
|
---|
| 355 |
|
---|
[4567] | 356 | results.Add("Execution Time", new TimeSpanValue(this.ExecutionTime));
|
---|
[4561] | 357 | results.Add("CrossValidation Folds", new RunCollection(runs));
|
---|
| 358 | }
|
---|
| 359 |
|
---|
| 360 | private static IEnumerable<IResult> ExtractAndAggregateResults<T>(IEnumerable<KeyValuePair<string, IItem>> results)
|
---|
| 361 | where T : class, IItem, new() {
|
---|
| 362 | Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
|
---|
| 363 | foreach (var resultValue in results.Where(r => r.Value.GetType() == typeof(T))) {
|
---|
| 364 | if (!resultValues.ContainsKey(resultValue.Key))
|
---|
| 365 | resultValues[resultValue.Key] = new List<double>();
|
---|
| 366 | resultValues[resultValue.Key].Add(ConvertToDouble(resultValue.Value));
|
---|
| 367 | }
|
---|
| 368 |
|
---|
| 369 | DoubleValue doubleValue;
|
---|
| 370 | if (typeof(T) == typeof(PercentValue))
|
---|
| 371 | doubleValue = new PercentValue();
|
---|
| 372 | else if (typeof(T) == typeof(DoubleValue))
|
---|
| 373 | doubleValue = new DoubleValue();
|
---|
| 374 | else if (typeof(T) == typeof(IntValue))
|
---|
| 375 | doubleValue = new DoubleValue();
|
---|
| 376 | else
|
---|
| 377 | throw new NotSupportedException();
|
---|
| 378 |
|
---|
| 379 | List<IResult> aggregatedResults = new List<IResult>();
|
---|
| 380 | foreach (KeyValuePair<string, List<double>> resultValue in resultValues) {
|
---|
| 381 | doubleValue.Value = resultValue.Value.Average();
|
---|
| 382 | aggregatedResults.Add(new Result(resultValue.Key, (IItem)doubleValue.Clone()));
|
---|
| 383 | doubleValue.Value = resultValue.Value.StandardDeviation();
|
---|
| 384 | aggregatedResults.Add(new Result(resultValue.Key + " StdDev", (IItem)doubleValue.Clone()));
|
---|
| 385 | }
|
---|
| 386 | return aggregatedResults;
|
---|
| 387 | }
|
---|
| 388 |
|
---|
| 389 | private static double ConvertToDouble(IItem item) {
|
---|
| 390 | if (item is DoubleValue) return ((DoubleValue)item).Value;
|
---|
| 391 | else if (item is IntValue) return ((IntValue)item).Value;
|
---|
| 392 | else throw new NotSupportedException("Could not convert any item type to double");
|
---|
| 393 | }
|
---|
| 394 | #endregion
|
---|
| 395 |
|
---|
[4472] | 396 | #region events
|
---|
[4542] | 397 | private void RegisterEvents() {
|
---|
| 398 | Folds.ValueChanged += new EventHandler(Folds_ValueChanged);
|
---|
[4561] | 399 | SamplesStart.ValueChanged += new EventHandler(SamplesStart_ValueChanged);
|
---|
| 400 | SamplesEnd.ValueChanged += new EventHandler(SamplesEnd_ValueChanged);
|
---|
[4542] | 401 | RegisterClonedAlgorithmsEvents();
|
---|
| 402 | }
|
---|
| 403 | private void Folds_ValueChanged(object sender, EventArgs e) {
|
---|
| 404 | if (ExecutionState != ExecutionState.Prepared)
|
---|
| 405 | throw new InvalidOperationException("Can not change number of folds if the execution state is not prepared.");
|
---|
| 406 | }
|
---|
[4561] | 407 | private void SamplesStart_ValueChanged(object sender, EventArgs e) {
|
---|
| 408 | if (Problem != null) Problem.DataAnalysisProblemData.TrainingSamplesStart.Value = SamplesStart.Value;
|
---|
[4542] | 409 | }
|
---|
[4561] | 410 | private void SamplesEnd_ValueChanged(object sender, EventArgs e) {
|
---|
| 411 | if (Problem != null) Problem.DataAnalysisProblemData.TrainingSamplesEnd.Value = SamplesEnd.Value;
|
---|
| 412 | }
|
---|
[4542] | 413 |
|
---|
| 414 | #region template algorithms events
|
---|
[4536] | 415 | public event EventHandler AlgorithmChanged;
|
---|
| 416 | private void OnAlgorithmChanged() {
|
---|
| 417 | EventHandler handler = AlgorithmChanged;
|
---|
[4472] | 418 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
[4536] | 419 | OnProblemChanged();
|
---|
[4542] | 420 | if (Problem == null) ExecutionState = ExecutionState.Stopped;
|
---|
[4472] | 421 | }
|
---|
[4536] | 422 | private void RegisterAlgorithmEvents() {
|
---|
| 423 | algorithm.ProblemChanged += new EventHandler(Algorithm_ProblemChanged);
|
---|
[4542] | 424 | algorithm.ExecutionStateChanged += new EventHandler(Algorithm_ExecutionStateChanged);
|
---|
[4472] | 425 | }
|
---|
[4536] | 426 | private void DeregisterAlgorithmEvents() {
|
---|
| 427 | algorithm.ProblemChanged -= new EventHandler(Algorithm_ProblemChanged);
|
---|
[4542] | 428 | algorithm.ExecutionStateChanged -= new EventHandler(Algorithm_ExecutionStateChanged);
|
---|
[4472] | 429 | }
|
---|
[4536] | 430 | private void Algorithm_ProblemChanged(object sender, EventArgs e) {
|
---|
[4563] | 431 | if (algorithm.Problem != null && !(algorithm.Problem is ISingleObjectiveDataAnalysisProblem)) {
|
---|
| 432 | algorithm.Problem = problem;
|
---|
[4536] | 433 | throw new ArgumentException("A cross validation algorithm can only contain DataAnalysisProblems.");
|
---|
| 434 | }
|
---|
[4563] | 435 | problem = (ISingleObjectiveDataAnalysisProblem)algorithm.Problem;
|
---|
[4536] | 436 | OnProblemChanged();
|
---|
[4472] | 437 | }
|
---|
[4536] | 438 | public event EventHandler ProblemChanged;
|
---|
| 439 | private void OnProblemChanged() {
|
---|
| 440 | EventHandler handler = ProblemChanged;
|
---|
| 441 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
[4561] | 442 |
|
---|
| 443 | SamplesStart.Value = 0;
|
---|
| 444 | if (Problem != null)
|
---|
| 445 | SamplesEnd.Value = Problem.DataAnalysisProblemData.Dataset.Rows;
|
---|
| 446 | else
|
---|
| 447 | SamplesEnd.Value = 0;
|
---|
[4536] | 448 | }
|
---|
[4472] | 449 |
|
---|
[4542] | 450 | private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
|
---|
| 451 | switch (Algorithm.ExecutionState) {
|
---|
| 452 | case ExecutionState.Prepared: OnPrepared();
|
---|
| 453 | break;
|
---|
| 454 | case ExecutionState.Started: throw new InvalidOperationException("Algorithm template can not be started.");
|
---|
| 455 | case ExecutionState.Paused: throw new InvalidOperationException("Algorithm template can not be paused.");
|
---|
| 456 | case ExecutionState.Stopped: OnStopped();
|
---|
| 457 | break;
|
---|
| 458 | }
|
---|
| 459 | }
|
---|
| 460 | #endregion
|
---|
| 461 |
|
---|
| 462 | #region clonedAlgorithms events
|
---|
| 463 | private void RegisterClonedAlgorithmsEvents() {
|
---|
[4472] | 464 | clonedAlgorithms.ItemsAdded += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
|
---|
| 465 | clonedAlgorithms.ItemsRemoved += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
|
---|
| 466 | clonedAlgorithms.CollectionReset += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
|
---|
[4542] | 467 | foreach (IAlgorithm algorithm in clonedAlgorithms)
|
---|
| 468 | RegisterClonedAlgorithmEvents(algorithm);
|
---|
[4472] | 469 | }
|
---|
[4561] | 470 | private void DeregisterClonedAlgorithmsEvents() {
|
---|
| 471 | clonedAlgorithms.ItemsAdded -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
|
---|
| 472 | clonedAlgorithms.ItemsRemoved -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
|
---|
| 473 | clonedAlgorithms.CollectionReset -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
|
---|
| 474 | foreach (IAlgorithm algorithm in clonedAlgorithms)
|
---|
| 475 | DeregisterClonedAlgorithmEvents(algorithm);
|
---|
| 476 | }
|
---|
[4472] | 477 | private void ClonedAlgorithms_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
|
---|
| 478 | foreach (IAlgorithm algorithm in e.Items)
|
---|
| 479 | RegisterClonedAlgorithmEvents(algorithm);
|
---|
| 480 | }
|
---|
| 481 | private void ClonedAlgorithms_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
|
---|
| 482 | foreach (IAlgorithm algorithm in e.Items)
|
---|
| 483 | DeregisterClonedAlgorithmEvents(algorithm);
|
---|
| 484 | }
|
---|
| 485 | private void ClonedAlgorithms_CollectionReset(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
|
---|
[4542] | 486 | foreach (IAlgorithm algorithm in e.OldItems)
|
---|
| 487 | DeregisterClonedAlgorithmEvents(algorithm);
|
---|
[4472] | 488 | foreach (IAlgorithm algorithm in e.Items)
|
---|
| 489 | RegisterClonedAlgorithmEvents(algorithm);
|
---|
| 490 | }
|
---|
| 491 | private void RegisterClonedAlgorithmEvents(IAlgorithm algorithm) {
|
---|
[4542] | 492 | algorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
|
---|
| 493 | algorithm.ExecutionTimeChanged += new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
|
---|
| 494 | algorithm.Started += new EventHandler(ClonedAlgorithm_Started);
|
---|
| 495 | algorithm.Paused += new EventHandler(ClonedAlgorithm_Paused);
|
---|
| 496 | algorithm.Stopped += new EventHandler(ClonedAlgorithm_Stopped);
|
---|
[4472] | 497 | }
|
---|
| 498 | private void DeregisterClonedAlgorithmEvents(IAlgorithm algorithm) {
|
---|
[4542] | 499 | algorithm.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
|
---|
| 500 | algorithm.ExecutionTimeChanged -= new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
|
---|
| 501 | algorithm.Started -= new EventHandler(ClonedAlgorithm_Started);
|
---|
| 502 | algorithm.Paused -= new EventHandler(ClonedAlgorithm_Paused);
|
---|
| 503 | algorithm.Stopped -= new EventHandler(ClonedAlgorithm_Stopped);
|
---|
[4472] | 504 | }
|
---|
[4542] | 505 | private void ClonedAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
|
---|
[4536] | 506 | OnExceptionOccurred(e.Value);
|
---|
| 507 | }
|
---|
[4542] | 508 | private void ClonedAlgorithm_ExecutionTimeChanged(object sender, EventArgs e) {
|
---|
[4536] | 509 | OnExecutionTimeChanged();
|
---|
| 510 | }
|
---|
[4542] | 511 |
|
---|
| 512 | private readonly object locker = new object();
|
---|
| 513 | private void ClonedAlgorithm_Started(object sender, EventArgs e) {
|
---|
| 514 | lock (locker) {
|
---|
[4567] | 515 | IAlgorithm algorithm = sender as IAlgorithm;
|
---|
| 516 | if (algorithm != null && !results.ContainsKey(algorithm.Name))
|
---|
| 517 | results.Add(new Result(algorithm.Name, "Contains results for the specific fold.", algorithm.Results));
|
---|
| 518 |
|
---|
[4542] | 519 | if (startPending) {
|
---|
| 520 | int startedAlgorithms = clonedAlgorithms.Count(alg => alg.ExecutionState == ExecutionState.Started);
|
---|
| 521 | if (startedAlgorithms == NumberOfWorkers.Value ||
|
---|
| 522 | clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Prepared))
|
---|
| 523 | startPending = false;
|
---|
| 524 |
|
---|
| 525 | if (pausePending) PauseAllClonedAlgorithms();
|
---|
| 526 | if (stopPending) StopAllClonedAlgorithms();
|
---|
| 527 | }
|
---|
| 528 | }
|
---|
[4472] | 529 | }
|
---|
[4542] | 530 |
|
---|
| 531 | private void ClonedAlgorithm_Paused(object sender, EventArgs e) {
|
---|
| 532 | lock (locker) {
|
---|
| 533 | if (pausePending && clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started))
|
---|
| 534 | OnPaused();
|
---|
| 535 | }
|
---|
[4472] | 536 | }
|
---|
[4542] | 537 |
|
---|
| 538 | private void ClonedAlgorithm_Stopped(object sender, EventArgs e) {
|
---|
| 539 | lock (locker) {
|
---|
| 540 | if (!stopPending && ExecutionState == ExecutionState.Started) {
|
---|
| 541 | IAlgorithm preparedAlgorithm = clonedAlgorithms.Where(alg => alg.ExecutionState == ExecutionState.Prepared ||
|
---|
| 542 | alg.ExecutionState == ExecutionState.Paused).FirstOrDefault();
|
---|
| 543 | if (preparedAlgorithm != null) preparedAlgorithm.Start();
|
---|
| 544 | }
|
---|
[4561] | 545 | if (ExecutionState != ExecutionState.Stopped) {
|
---|
| 546 | if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped))
|
---|
| 547 | OnStopped();
|
---|
| 548 | else if (stopPending &&
|
---|
| 549 | clonedAlgorithms.All(
|
---|
| 550 | alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped))
|
---|
| 551 | OnStopped();
|
---|
| 552 | }
|
---|
[4542] | 553 | }
|
---|
[4472] | 554 | }
|
---|
[4542] | 555 | #endregion
|
---|
| 556 | #endregion
|
---|
| 557 |
|
---|
| 558 | #region event firing
|
---|
[4472] | 559 | public event EventHandler ExecutionStateChanged;
|
---|
| 560 | private void OnExecutionStateChanged() {
|
---|
| 561 | EventHandler handler = ExecutionStateChanged;
|
---|
| 562 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
| 563 | }
|
---|
| 564 | public event EventHandler ExecutionTimeChanged;
|
---|
| 565 | private void OnExecutionTimeChanged() {
|
---|
| 566 | EventHandler handler = ExecutionTimeChanged;
|
---|
| 567 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
| 568 | }
|
---|
| 569 | public event EventHandler Prepared;
|
---|
| 570 | private void OnPrepared() {
|
---|
| 571 | ExecutionState = ExecutionState.Prepared;
|
---|
| 572 | EventHandler handler = Prepared;
|
---|
| 573 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
[4567] | 574 | OnExecutionTimeChanged();
|
---|
[4472] | 575 | }
|
---|
| 576 | public event EventHandler Started;
|
---|
| 577 | private void OnStarted() {
|
---|
[4542] | 578 | startPending = false;
|
---|
[4472] | 579 | ExecutionState = ExecutionState.Started;
|
---|
| 580 | EventHandler handler = Started;
|
---|
| 581 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
| 582 | }
|
---|
| 583 | public event EventHandler Paused;
|
---|
| 584 | private void OnPaused() {
|
---|
[4542] | 585 | pausePending = false;
|
---|
[4472] | 586 | ExecutionState = ExecutionState.Paused;
|
---|
| 587 | EventHandler handler = Paused;
|
---|
| 588 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
| 589 | }
|
---|
| 590 | public event EventHandler Stopped;
|
---|
| 591 | private void OnStopped() {
|
---|
[4542] | 592 | stopPending = false;
|
---|
[4567] | 593 | Dictionary<string, IItem> collectedResults = new Dictionary<string, IItem>();
|
---|
| 594 | CollectResultValues(collectedResults);
|
---|
| 595 | results.AddRange(collectedResults.Select(x => new Result(x.Key, x.Value)).Cast<IResult>().ToArray());
|
---|
[4561] | 596 | runsCounter++;
|
---|
| 597 | runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
|
---|
[4472] | 598 | ExecutionState = ExecutionState.Stopped;
|
---|
| 599 | EventHandler handler = Stopped;
|
---|
| 600 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
| 601 | }
|
---|
| 602 | public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
|
---|
| 603 | private void OnExceptionOccurred(Exception exception) {
|
---|
| 604 | EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
|
---|
| 605 | if (handler != null) handler(this, new EventArgs<Exception>(exception));
|
---|
| 606 | }
|
---|
[4561] | 607 | public event EventHandler StoreAlgorithmInEachRunChanged;
|
---|
| 608 | private void OnStoreAlgorithmInEachRunChanged() {
|
---|
| 609 | EventHandler handler = StoreAlgorithmInEachRunChanged;
|
---|
| 610 | if (handler != null) handler(this, EventArgs.Empty);
|
---|
| 611 | }
|
---|
[4472] | 612 | #endregion
|
---|
| 613 | }
|
---|
| 614 | }
|
---|