source: branches/2965_CancelablePersistence/HeuristicLab.Algorithms.Benchmarks/3.3/BenchmarkAlgorithm.cs @ 16433

Last change on this file since 16433 was 16433, checked in by pfleck, 8 months ago

#2965 Merged recent trunk changes.
Enabled the prepared hooks that allows to cancel the save file using the recently introduced cancelable progressbars (in FileManager).

File size: 19.2 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Drawing;
25using System.Linq;
26using System.Threading;
27using System.Threading.Tasks;
28using HeuristicLab.Collections;
29using HeuristicLab.Common;
30using HeuristicLab.Core;
31using HeuristicLab.Data;
32using HeuristicLab.Optimization;
33using HeuristicLab.Parameters;
34using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
35using HeuristicLab.PluginInfrastructure;
36
37namespace HeuristicLab.Algorithms.Benchmarks {
38  [Item("Benchmark Algorithm", "An algorithm to execute performance benchmarks (Linpack, Dhrystone, Whetstone, etc.).")]
39  [Creatable(CreatableAttribute.Categories.TestingAndAnalysis, Priority = 130)]
40  [StorableClass]
41  public sealed class BenchmarkAlgorithm : IAlgorithm, IStorableContent {
42    private CancellationTokenSource cancellationTokenSource;
43
44    public string Filename { get; set; }
45
46    public string ItemName {
47      get { return ItemAttribute.GetName(this.GetType()); }
48    }
49    public string ItemDescription {
50      get { return ItemAttribute.GetDescription(this.GetType()); }
51    }
52    public Version ItemVersion {
53      get { return ItemAttribute.GetVersion(this.GetType()); }
54    }
55    public static Image StaticItemImage {
56      get { return HeuristicLab.Common.Resources.VSImageLibrary.Event; }
57    }
58    public Image ItemImage {
59      get {
60        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
61        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
62        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
63        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
64        else return ItemAttribute.GetImage(this.GetType());
65      }
66    }
67
68    [Storable]
69    private DateTime lastUpdateTime;
70
71    [Storable]
72    private IBenchmark benchmark;
73    public IBenchmark Benchmark {
74      get { return benchmark; }
75      set {
76        if (value == null) throw new ArgumentNullException();
77        benchmark = value;
78      }
79    }
80
81    [Storable]
82    private ExecutionState executionState;
83    public ExecutionState ExecutionState {
84      get { return executionState; }
85      private set {
86        if (executionState != value) {
87          executionState = value;
88          OnExecutionStateChanged();
89          OnItemImageChanged();
90        }
91      }
92    }
93
94    [Storable]
95    private TimeSpan executionTime;
96    public TimeSpan ExecutionTime {
97      get { return executionTime; }
98      private set {
99        executionTime = value;
100        OnExecutionTimeChanged();
101      }
102    }
103
104    [Storable]
105    private bool storeAlgorithmInEachRun;
106    public bool StoreAlgorithmInEachRun {
107      get { return storeAlgorithmInEachRun; }
108      set {
109        if (storeAlgorithmInEachRun != value) {
110          storeAlgorithmInEachRun = value;
111          OnStoreAlgorithmInEachRunChanged();
112        }
113      }
114    }
115
116    [Storable]
117    private int runsCounter;
118
119    [Storable]
120    private RunCollection runs = new RunCollection();
121    public RunCollection Runs {
122      get { return runs; }
123      private set {
124        if (value == null) throw new ArgumentNullException();
125        if (runs != value) {
126          if (runs != null) DeregisterRunsEvents();
127          runs = value;
128          if (runs != null) RegisterRunsEvents();
129        }
130      }
131    }
132
133    [Storable]
134    private ResultCollection results;
135    public ResultCollection Results {
136      get { return results; }
137    }
138
139    public Type ProblemType {
140      get {
141        // BenchmarkAlgorithm does not have a problem, so return a type which is no problem for sure
142        return typeof(BenchmarkAlgorithm);
143      }
144    }
145
146    public IProblem Problem {
147      get { return null; }
148      set { throw new NotImplementedException("BenchmarkAlgorithm does not have a problem."); }
149    }
150
151    [Storable]
152    private string name;
153    public string Name {
154      get { return name; }
155      set {
156        if (!CanChangeName) throw new NotSupportedException("Name cannot be changed.");
157        if (!(name.Equals(value) || (value == null) && (name == string.Empty))) {
158          CancelEventArgs<string> e = value == null ? new CancelEventArgs<string>(string.Empty) : new CancelEventArgs<string>(value);
159          OnNameChanging(e);
160          if (!e.Cancel) {
161            name = value == null ? string.Empty : value;
162            OnNameChanged();
163            runs.OptimizerName = name;
164          }
165        }
166      }
167    }
168    public bool CanChangeName {
169      get { return true; }
170    }
171
172    [Storable]
173    private string description;
174    public string Description {
175      get { return description; }
176      set {
177        if (!CanChangeDescription) throw new NotSupportedException("Description cannot be changed.");
178        if (!(description.Equals(value) || (value == null) && (description == string.Empty))) {
179          description = value == null ? string.Empty : value;
180          OnDescriptionChanged();
181        }
182      }
183    }
184    public bool CanChangeDescription {
185      get { return true; }
186    }
187
188    [Storable]
189    private ParameterCollection parameters = new ParameterCollection();
190    public IKeyedItemCollection<string, IParameter> Parameters {
191      get { return parameters; }
192    }
193    private ReadOnlyKeyedItemCollection<string, IParameter> readOnlyParameters;
194    IKeyedItemCollection<string, IParameter> IParameterizedItem.Parameters {
195      get {
196        if (readOnlyParameters == null) readOnlyParameters = parameters.AsReadOnly();
197        return readOnlyParameters;
198      }
199    }
200
201    public IEnumerable<IOptimizer> NestedOptimizers {
202      get { return Enumerable.Empty<IOptimizer>(); }
203    }
204
205    #region Parameter Properties
206    public IConstrainedValueParameter<IBenchmark> BenchmarkParameter {
207      get { return (IConstrainedValueParameter<IBenchmark>)Parameters["Benchmark"]; }
208    }
209    private ValueParameter<IntValue> ChunkSizeParameter {
210      get { return (ValueParameter<IntValue>)Parameters["ChunkSize"]; }
211    }
212    private ValueParameter<DoubleValue> TimeLimitParameter {
213      get { return (ValueParameter<DoubleValue>)Parameters["TimeLimit"]; }
214    }
215    #endregion
216
217    #region Constructors
218    [StorableConstructor]
219    private BenchmarkAlgorithm(bool deserializing) { }
220    private BenchmarkAlgorithm(BenchmarkAlgorithm original, Cloner cloner) {
221      if (original.ExecutionState == ExecutionState.Started) throw new InvalidOperationException(string.Format("Clone not allowed in execution state \"{0}\".", ExecutionState));
222      cloner.RegisterClonedObject(original, this);
223      name = original.name;
224      description = original.description;
225      parameters = cloner.Clone(original.parameters);
226      readOnlyParameters = null;
227      executionState = original.executionState;
228      executionTime = original.executionTime;
229      storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
230      runsCounter = original.runsCounter;
231      Runs = cloner.Clone(original.runs);
232      results = cloner.Clone(original.results);
233    }
234    public BenchmarkAlgorithm() {
235      name = ItemName;
236      description = ItemDescription;
237      parameters = new ParameterCollection();
238      readOnlyParameters = null;
239      executionState = ExecutionState.Stopped;
240      executionTime = TimeSpan.Zero;
241      storeAlgorithmInEachRun = false;
242      runsCounter = 0;
243      Runs = new RunCollection() { OptimizerName = name };
244      results = new ResultCollection();
245      CreateParameters();
246      DiscoverBenchmarks();
247      Prepare();
248    }
249    #endregion
250
251    private void CreateParameters() {
252      Parameters.Add(new ValueParameter<IntValue>("ChunkSize", "The size in MB of the chunk data array that is generated.", new IntValue(0)));
253      Parameters.Add(new ValueParameter<DoubleValue>("TimeLimit", "The time limit in minutes for a benchmark run (zero means a fixed number of iterations).", new DoubleValue(0)));
254    }
255    private void DiscoverBenchmarks() {
256      var benchmarks = from t in ApplicationManager.Manager.GetTypes(typeof(IBenchmark))
257                       select t;
258      ItemSet<IBenchmark> values = new ItemSet<IBenchmark>();
259      foreach (var benchmark in benchmarks) {
260        IBenchmark b = (IBenchmark)Activator.CreateInstance(benchmark);
261        values.Add(b);
262      }
263      string paramName = "Benchmark";
264      if (!Parameters.ContainsKey(paramName)) {
265        if (values.Count > 0) {
266          Parameters.Add(new ConstrainedValueParameter<IBenchmark>(paramName, "The benchmark which should be executed.", values, values.First(a => a is IBenchmark)));
267        } else {
268          Parameters.Add(new ConstrainedValueParameter<IBenchmark>(paramName, "The benchmark which should be executed.", values));
269        }
270      }
271    }
272
273    public IDeepCloneable Clone(Cloner cloner) {
274      return new BenchmarkAlgorithm(this, cloner);
275    }
276    public object Clone() {
277      return Clone(new Cloner());
278    }
279
280    public override string ToString() {
281      return Name;
282    }
283
284    public void Prepare() {
285      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused) && (ExecutionState != ExecutionState.Stopped))
286        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
287      results.Clear();
288      OnPrepared();
289    }
290    public void Prepare(bool clearRuns) {
291      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused) && (ExecutionState != ExecutionState.Stopped))
292        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
293      if (clearRuns) runs.Clear();
294      Prepare();
295    }
296    public void Pause() {
297      if (ExecutionState != ExecutionState.Started)
298        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
299    }
300    public void Stop() {
301      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
302        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".", ExecutionState));
303      cancellationTokenSource.Cancel();
304    }
305    public void Start() {
306      Start(CancellationToken.None);
307    }
308    public void Start(CancellationToken cancellationToken) {
309      cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
310      OnStarted();
311
312      try {
313        Run(cancellationTokenSource.Token);
314      } catch (OperationCanceledException) {
315      } catch (AggregateException ae) {
316        ae.FlattenAndHandle(new[] { typeof(OperationCanceledException) }, e => OnExceptionOccurred(e));
317      } catch (Exception e) {
318        OnExceptionOccurred(e);
319      }
320
321      cancellationTokenSource.Dispose();
322      cancellationTokenSource = null;
323      OnStopped();
324    }
325    public async Task StartAsync() { await StartAsync(CancellationToken.None); }
326    public async Task StartAsync(CancellationToken cancellationToken) {
327      await AsyncHelper.DoAsync(Start, cancellationToken);
328    }
329
330    private void Run(object state) {
331      CancellationToken cancellationToken = (CancellationToken)state;
332      lastUpdateTime = DateTime.UtcNow;
333      System.Timers.Timer timer = new System.Timers.Timer(250);
334      timer.AutoReset = true;
335      timer.Elapsed += new System.Timers.ElapsedEventHandler(timer_Elapsed);
336      timer.Start();
337      try {
338        Benchmark = (IBenchmark)BenchmarkParameter.ActualValue;
339        int chunkSize = ((IntValue)ChunkSizeParameter.ActualValue).Value;
340        if (chunkSize > 0) {
341          Benchmark.ChunkData = CreateChunkData(chunkSize);
342        } else if (chunkSize < 0) {
343          throw new ArgumentException("ChunkSize must not be negativ.");
344        }
345        TimeSpan timelimit = TimeSpan.FromMinutes(((DoubleValue)TimeLimitParameter.ActualValue).Value);
346        if (timelimit.TotalMilliseconds < 0) {
347          throw new ArgumentException("TimeLimit must not be negativ. ");
348        }
349        Benchmark.TimeLimit = timelimit;
350        Benchmark.Run(cancellationToken, results);
351      } catch (OperationCanceledException) {
352      } finally {
353        timer.Elapsed -= new System.Timers.ElapsedEventHandler(timer_Elapsed);
354        timer.Stop();
355        ExecutionTime += DateTime.UtcNow - lastUpdateTime;
356      }
357    }
358
359    private void timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) {
360      System.Timers.Timer timer = (System.Timers.Timer)sender;
361      timer.Enabled = false;
362      DateTime now = DateTime.UtcNow;
363      ExecutionTime += now - lastUpdateTime;
364      lastUpdateTime = now;
365      timer.Enabled = true;
366    }
367
368    public void CollectResultValues(IDictionary<string, IItem> values) {
369      values.Add("Execution Time", new TimeSpanValue(ExecutionTime));
370      CollectResultsRecursively("", Results, values);
371    }
372    private void CollectResultsRecursively(string path, ResultCollection results, IDictionary<string, IItem> values) {
373      foreach (IResult result in results) {
374        values.Add(path + result.Name, result.Value);
375        ResultCollection childCollection = result.Value as ResultCollection;
376        if (childCollection != null) {
377          CollectResultsRecursively(path + result.Name + ".", childCollection, values);
378        }
379      }
380    }
381    public void CollectParameterValues(IDictionary<string, IItem> values) {
382      foreach (IValueParameter param in parameters.OfType<IValueParameter>()) {
383        if (param.GetsCollected && param.Value != null) values.Add(param.Name, param.Value);
384        if (param.Value is IParameterizedItem) {
385          Dictionary<string, IItem> children = new Dictionary<string, IItem>();
386          ((IParameterizedItem)param.Value).CollectParameterValues(children);
387          foreach (string key in children.Keys)
388            values.Add(param.Name + "." + key, children[key]);
389        }
390      }
391    }
392
393    private byte[][] CreateChunkData(int megaBytes) {
394      if (megaBytes <= 0) {
395        throw new ArgumentException("MegaBytes must be greater than zero", "megaBytes");
396      }
397      Random random = new Random();
398      byte[][] chunk = new byte[megaBytes][];
399      for (int i = 0; i < chunk.Length; i++) {
400        chunk[i] = new byte[1024 * 1024];
401        random.NextBytes(chunk[i]);
402      }
403      return chunk;
404    }
405
406    #region Events
407    public event EventHandler ExecutionStateChanged;
408    private void OnExecutionStateChanged() {
409      EventHandler handler = ExecutionStateChanged;
410      if (handler != null) handler(this, EventArgs.Empty);
411    }
412    public event EventHandler ExecutionTimeChanged;
413    private void OnExecutionTimeChanged() {
414      EventHandler handler = ExecutionTimeChanged;
415      if (handler != null) handler(this, EventArgs.Empty);
416    }
417    public event EventHandler ProblemChanged { add { } remove { } }
418    public event EventHandler StoreAlgorithmInEachRunChanged;
419    private void OnStoreAlgorithmInEachRunChanged() {
420      EventHandler handler = StoreAlgorithmInEachRunChanged;
421      if (handler != null) handler(this, EventArgs.Empty);
422    }
423    public event EventHandler Prepared;
424    private void OnPrepared() {
425      ExecutionState = ExecutionState.Prepared;
426      ExecutionTime = TimeSpan.Zero;
427      foreach (IStatefulItem statefulObject in this.GetObjectGraphObjects().OfType<IStatefulItem>()) {
428        statefulObject.InitializeState();
429      }
430      EventHandler handler = Prepared;
431      if (handler != null) handler(this, EventArgs.Empty);
432    }
433    public event EventHandler Started;
434    private void OnStarted() {
435      ExecutionState = ExecutionState.Started;
436      EventHandler handler = Started;
437      if (handler != null) handler(this, EventArgs.Empty);
438    }
439    public event EventHandler Paused;
440    private void OnPaused() {
441      ExecutionState = ExecutionState.Paused;
442      EventHandler handler = Paused;
443      if (handler != null) handler(this, EventArgs.Empty);
444    }
445    public event EventHandler Stopped;
446    private void OnStopped() {
447      ExecutionState = ExecutionState.Stopped;
448      foreach (IStatefulItem statefulObject in this.GetObjectGraphObjects().OfType<IStatefulItem>()) {
449        statefulObject.ClearState();
450      }
451      runsCounter++;
452      runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
453      EventHandler handler = Stopped;
454      if (handler != null) handler(this, EventArgs.Empty);
455    }
456    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
457    private void OnExceptionOccurred(Exception exception) {
458      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
459      if (handler != null) handler(this, new EventArgs<Exception>(exception));
460    }
461
462    public event EventHandler<CancelEventArgs<string>> NameChanging;
463    private void OnNameChanging(CancelEventArgs<string> e) {
464      var handler = NameChanging;
465      if (handler != null) handler(this, e);
466    }
467
468    public event EventHandler NameChanged;
469    private void OnNameChanged() {
470      var handler = NameChanged;
471      if (handler != null) handler(this, EventArgs.Empty);
472      OnToStringChanged();
473    }
474
475    public event EventHandler DescriptionChanged;
476    private void OnDescriptionChanged() {
477      var handler = DescriptionChanged;
478      if (handler != null) handler(this, EventArgs.Empty);
479    }
480
481    public event EventHandler ItemImageChanged;
482    private void OnItemImageChanged() {
483      EventHandler handler = ItemImageChanged;
484      if (handler != null) handler(this, EventArgs.Empty);
485    }
486    public event EventHandler ToStringChanged;
487    private void OnToStringChanged() {
488      EventHandler handler = ToStringChanged;
489      if (handler != null) handler(this, EventArgs.Empty);
490    }
491
492    private void DeregisterRunsEvents() {
493      runs.CollectionReset -= new CollectionItemsChangedEventHandler<IRun>(Runs_CollectionReset);
494    }
495    private void RegisterRunsEvents() {
496      runs.CollectionReset += new CollectionItemsChangedEventHandler<IRun>(Runs_CollectionReset);
497    }
498    private void Runs_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
499      runsCounter = runs.Count;
500    }
501    #endregion
502  }
503}
Note: See TracBrowser for help on using the repository browser.