Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/sources/HeuristicLab.Algorithms.Benchmarks/3.3/BenchmarkAlgorithm.cs @ 12504

Last change on this file since 12504 was 12504, checked in by mkommend, 9 years ago

#2025: Changed categories for all creatables.

File size: 19.0 KB
RevLine 
[6934]1#region License Information
2/* HeuristicLab
[12012]3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
[6934]4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Drawing;
25using System.Linq;
26using System.Threading;
27using System.Threading.Tasks;
28using HeuristicLab.Collections;
29using HeuristicLab.Common;
30using HeuristicLab.Core;
31using HeuristicLab.Data;
32using HeuristicLab.Optimization;
[6948]33using HeuristicLab.Parameters;
[6934]34using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
[6948]35using HeuristicLab.PluginInfrastructure;
[6934]36
37namespace HeuristicLab.Algorithms.Benchmarks {
[7246]38  [Item("Benchmark Algorithm", "An algorithm to execute performance benchmarks (Linpack, Dhrystone, Whetstone, etc.).")]
[12504]39  [Creatable(CreatableAttribute.Categories.TestingAndAnalysis, Priority = 130)]
[6934]40  [StorableClass]
[7246]41  public sealed class BenchmarkAlgorithm : IAlgorithm {
42    private CancellationTokenSource cancellationTokenSource;
[6934]43
[7248]44    public string ItemName {
45      get { return ItemAttribute.GetName(this.GetType()); }
46    }
47    public string ItemDescription {
48      get { return ItemAttribute.GetDescription(this.GetType()); }
49    }
50    public Version ItemVersion {
51      get { return ItemAttribute.GetVersion(this.GetType()); }
52    }
53    public static Image StaticItemImage {
54      get { return HeuristicLab.Common.Resources.VSImageLibrary.Event; }
55    }
56    public Image ItemImage {
57      get {
58        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
59        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
60        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
61        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
62        else return ItemAttribute.GetImage(this.GetType());
63      }
64    }
65
[6948]66    [Storable]
[6934]67    private DateTime lastUpdateTime;
68
69    [Storable]
[7246]70    private IBenchmark benchmark;
71    public IBenchmark Benchmark {
72      get { return benchmark; }
[6948]73      set {
74        if (value == null) throw new ArgumentNullException();
[7246]75        benchmark = value;
[6948]76      }
77    }
78
79    [Storable]
[6934]80    private ExecutionState executionState;
81    public ExecutionState ExecutionState {
82      get { return executionState; }
83      private set {
84        if (executionState != value) {
85          executionState = value;
86          OnExecutionStateChanged();
87          OnItemImageChanged();
88        }
89      }
90    }
91
92    [Storable]
93    private TimeSpan executionTime;
94    public TimeSpan ExecutionTime {
95      get { return executionTime; }
[7246]96      private set {
[6934]97        executionTime = value;
98        OnExecutionTimeChanged();
99      }
100    }
101
102    [Storable]
103    private bool storeAlgorithmInEachRun;
104    public bool StoreAlgorithmInEachRun {
105      get { return storeAlgorithmInEachRun; }
106      set {
107        if (storeAlgorithmInEachRun != value) {
108          storeAlgorithmInEachRun = value;
109          OnStoreAlgorithmInEachRunChanged();
110        }
111      }
112    }
113
114    [Storable]
[7246]115    private int runsCounter;
[6934]116
117    [Storable]
118    private RunCollection runs = new RunCollection();
119    public RunCollection Runs {
120      get { return runs; }
[7246]121      private set {
[6934]122        if (value == null) throw new ArgumentNullException();
123        if (runs != value) {
124          if (runs != null) DeregisterRunsEvents();
125          runs = value;
126          if (runs != null) RegisterRunsEvents();
127        }
128      }
129    }
130
131    [Storable]
132    private ResultCollection results;
133    public ResultCollection Results {
134      get { return results; }
135    }
136
[7246]137    public Type ProblemType {
[7248]138      get {
139        // BenchmarkAlgorithm does not have a problem, so return a type which is no problem for sure
140        return typeof(BenchmarkAlgorithm);
141      }
[7246]142    }
143
[6934]144    public IProblem Problem {
[7248]145      get { return null; }
146      set { throw new NotImplementedException("BenchmarkAlgorithm does not have a problem."); }
[6934]147    }
148
149    [Storable]
[7246]150    private string name;
[6934]151    public string Name {
152      get { return name; }
153      set {
154        if (!CanChangeName) throw new NotSupportedException("Name cannot be changed.");
155        if (!(name.Equals(value) || (value == null) && (name == string.Empty))) {
156          CancelEventArgs<string> e = value == null ? new CancelEventArgs<string>(string.Empty) : new CancelEventArgs<string>(value);
157          OnNameChanging(e);
158          if (!e.Cancel) {
159            name = value == null ? string.Empty : value;
160            OnNameChanged();
[8962]161            runs.OptimizerName = name;
[6934]162          }
163        }
164      }
165    }
166    public bool CanChangeName {
[7248]167      get { return true; }
[6934]168    }
169
170    [Storable]
[7246]171    private string description;
[6934]172    public string Description {
173      get { return description; }
174      set {
175        if (!CanChangeDescription) throw new NotSupportedException("Description cannot be changed.");
176        if (!(description.Equals(value) || (value == null) && (description == string.Empty))) {
177          description = value == null ? string.Empty : value;
178          OnDescriptionChanged();
179        }
180      }
181    }
182    public bool CanChangeDescription {
[7248]183      get { return true; }
[6934]184    }
185
186    [Storable]
187    private ParameterCollection parameters = new ParameterCollection();
188    public IKeyedItemCollection<string, IParameter> Parameters {
189      get { return parameters; }
190    }
191    private ReadOnlyKeyedItemCollection<string, IParameter> readOnlyParameters;
192    IKeyedItemCollection<string, IParameter> IParameterizedItem.Parameters {
193      get {
194        if (readOnlyParameters == null) readOnlyParameters = parameters.AsReadOnly();
195        return readOnlyParameters;
196      }
197    }
198
199    public IEnumerable<IOptimizer> NestedOptimizers {
200      get { return Enumerable.Empty<IOptimizer>(); }
201    }
202
[6948]203    #region Parameter Properties
[8121]204    public IConstrainedValueParameter<IBenchmark> BenchmarkParameter {
205      get { return (IConstrainedValueParameter<IBenchmark>)Parameters["Benchmark"]; }
[6948]206    }
207    private ValueParameter<IntValue> ChunkSizeParameter {
208      get { return (ValueParameter<IntValue>)Parameters["ChunkSize"]; }
209    }
210    private ValueParameter<DoubleValue> TimeLimitParameter {
211      get { return (ValueParameter<DoubleValue>)Parameters["TimeLimit"]; }
212    }
213    #endregion
214
[6934]215    #region Constructors
[7002]216    [StorableConstructor]
[7248]217    private BenchmarkAlgorithm(bool deserializing) { }
218    private BenchmarkAlgorithm(BenchmarkAlgorithm original, Cloner cloner) {
219      if (original.ExecutionState == ExecutionState.Started) throw new InvalidOperationException(string.Format("Clone not allowed in execution state \"{0}\".", ExecutionState));
220      cloner.RegisterClonedObject(original, this);
221      name = original.name;
222      description = original.description;
223      parameters = cloner.Clone(original.parameters);
224      readOnlyParameters = null;
225      executionState = original.executionState;
226      executionTime = original.executionTime;
227      storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
228      runsCounter = original.runsCounter;
229      Runs = cloner.Clone(original.runs);
230      results = cloner.Clone(original.results);
231    }
[7246]232    public BenchmarkAlgorithm() {
[6934]233      name = ItemName;
234      description = ItemDescription;
235      parameters = new ParameterCollection();
236      readOnlyParameters = null;
237      executionState = ExecutionState.Stopped;
238      executionTime = TimeSpan.Zero;
239      storeAlgorithmInEachRun = false;
240      runsCounter = 0;
[8962]241      Runs = new RunCollection() { OptimizerName = name };
[6934]242      results = new ResultCollection();
[6948]243      CreateParameters();
244      DiscoverBenchmarks();
[6934]245      Prepare();
246    }
247    #endregion
248
[6948]249    private void CreateParameters() {
[7248]250      Parameters.Add(new ValueParameter<IntValue>("ChunkSize", "The size in MB of the chunk data array that is generated.", new IntValue(0)));
251      Parameters.Add(new ValueParameter<DoubleValue>("TimeLimit", "The time limit in minutes for a benchmark run (zero means a fixed number of iterations).", new DoubleValue(0)));
[6948]252    }
253    private void DiscoverBenchmarks() {
254      var benchmarks = from t in ApplicationManager.Manager.GetTypes(typeof(IBenchmark))
255                       select t;
256      ItemSet<IBenchmark> values = new ItemSet<IBenchmark>();
257      foreach (var benchmark in benchmarks) {
258        IBenchmark b = (IBenchmark)Activator.CreateInstance(benchmark);
259        values.Add(b);
260      }
[7246]261      string paramName = "Benchmark";
[6948]262      if (!Parameters.ContainsKey(paramName)) {
263        if (values.Count > 0) {
[7248]264          Parameters.Add(new ConstrainedValueParameter<IBenchmark>(paramName, "The benchmark which should be executed.", values, values.First(a => a is IBenchmark)));
[6948]265        } else {
[7248]266          Parameters.Add(new ConstrainedValueParameter<IBenchmark>(paramName, "The benchmark which should be executed.", values));
[6948]267        }
268      }
269    }
270
[7248]271    public IDeepCloneable Clone(Cloner cloner) {
272      return new BenchmarkAlgorithm(this, cloner);
[6934]273    }
[7248]274    public object Clone() {
275      return Clone(new Cloner());
276    }
[6934]277
[7248]278    public override string ToString() {
279      return Name;
280    }
281
[7246]282    public void Prepare() {
[6934]283      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused) && (ExecutionState != ExecutionState.Stopped))
284        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
285      results.Clear();
286      OnPrepared();
287    }
288    public void Prepare(bool clearRuns) {
289      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused) && (ExecutionState != ExecutionState.Stopped))
290        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
291      if (clearRuns) runs.Clear();
292      Prepare();
293    }
[7246]294    public void Pause() {
[6934]295      if (ExecutionState != ExecutionState.Started)
296        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
297    }
[7246]298    public void Stop() {
[6934]299      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
300        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".", ExecutionState));
[6948]301      cancellationTokenSource.Cancel();
[6934]302    }
[7246]303    public void Start() {
[6948]304      cancellationTokenSource = new CancellationTokenSource();
[6934]305      OnStarted();
306      Task task = Task.Factory.StartNew(Run, cancellationTokenSource.Token, cancellationTokenSource.Token);
307      task.ContinueWith(t => {
308        try {
309          t.Wait();
310        }
311        catch (AggregateException ex) {
312          try {
313            ex.Flatten().Handle(x => x is OperationCanceledException);
314          }
315          catch (AggregateException remaining) {
316            if (remaining.InnerExceptions.Count == 1) OnExceptionOccurred(remaining.InnerExceptions[0]);
317            else OnExceptionOccurred(remaining);
318          }
319        }
[6948]320
[6934]321        cancellationTokenSource.Dispose();
322        cancellationTokenSource = null;
323        OnStopped();
324      });
325    }
326
[7246]327    private void Run(object state) {
[6934]328      CancellationToken cancellationToken = (CancellationToken)state;
[9343]329      lastUpdateTime = DateTime.UtcNow;
[6934]330      System.Timers.Timer timer = new System.Timers.Timer(250);
331      timer.AutoReset = true;
332      timer.Elapsed += new System.Timers.ElapsedEventHandler(timer_Elapsed);
333      timer.Start();
334      try {
[7246]335        Benchmark = (IBenchmark)BenchmarkParameter.ActualValue;
[6948]336        int chunkSize = ((IntValue)ChunkSizeParameter.ActualValue).Value;
337        if (chunkSize > 0) {
[7248]338          Benchmark.ChunkData = CreateChunkData(chunkSize);
[6987]339        } else if (chunkSize < 0) {
340          throw new ArgumentException("ChunkSize must not be negativ.");
[6948]341        }
[6987]342        TimeSpan timelimit = TimeSpan.FromMinutes(((DoubleValue)TimeLimitParameter.ActualValue).Value);
343        if (timelimit.TotalMilliseconds < 0) {
344          throw new ArgumentException("TimeLimit must not be negativ. ");
345        }
[7246]346        Benchmark.TimeLimit = timelimit;
347        Benchmark.Run(cancellationToken, results);
[6934]348      }
[6948]349      catch (OperationCanceledException) {
350      }
[6934]351      finally {
352        timer.Elapsed -= new System.Timers.ElapsedEventHandler(timer_Elapsed);
353        timer.Stop();
[9343]354        ExecutionTime += DateTime.UtcNow - lastUpdateTime;
[6934]355      }
356    }
357
358    private void timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) {
359      System.Timers.Timer timer = (System.Timers.Timer)sender;
360      timer.Enabled = false;
[9343]361      DateTime now = DateTime.UtcNow;
[6934]362      ExecutionTime += now - lastUpdateTime;
363      lastUpdateTime = now;
364      timer.Enabled = true;
365    }
366
[7246]367    public void CollectResultValues(IDictionary<string, IItem> values) {
[6934]368      values.Add("Execution Time", new TimeSpanValue(ExecutionTime));
369      CollectResultsRecursively("", Results, values);
370    }
371    private void CollectResultsRecursively(string path, ResultCollection results, IDictionary<string, IItem> values) {
372      foreach (IResult result in results) {
373        values.Add(path + result.Name, result.Value);
374        ResultCollection childCollection = result.Value as ResultCollection;
375        if (childCollection != null) {
376          CollectResultsRecursively(path + result.Name + ".", childCollection, values);
377        }
378      }
379    }
[7246]380    public void CollectParameterValues(IDictionary<string, IItem> values) {
[6934]381      foreach (IValueParameter param in parameters.OfType<IValueParameter>()) {
382        if (param.GetsCollected && param.Value != null) values.Add(param.Name, param.Value);
383        if (param.Value is IParameterizedItem) {
384          Dictionary<string, IItem> children = new Dictionary<string, IItem>();
385          ((IParameterizedItem)param.Value).CollectParameterValues(children);
386          foreach (string key in children.Keys)
387            values.Add(param.Name + "." + key, children[key]);
388        }
389      }
390    }
391
[7248]392    private byte[][] CreateChunkData(int megaBytes) {
[6948]393      if (megaBytes <= 0) {
394        throw new ArgumentException("MegaBytes must be greater than zero", "megaBytes");
395      }
[7248]396      Random random = new Random();
[6948]397      byte[][] chunk = new byte[megaBytes][];
398      for (int i = 0; i < chunk.Length; i++) {
399        chunk[i] = new byte[1024 * 1024];
400        random.NextBytes(chunk[i]);
401      }
402      return chunk;
403    }
404
[6934]405    #region Events
406    public event EventHandler ExecutionStateChanged;
[7246]407    private void OnExecutionStateChanged() {
[6934]408      EventHandler handler = ExecutionStateChanged;
409      if (handler != null) handler(this, EventArgs.Empty);
410    }
411    public event EventHandler ExecutionTimeChanged;
[7246]412    private void OnExecutionTimeChanged() {
[6934]413      EventHandler handler = ExecutionTimeChanged;
414      if (handler != null) handler(this, EventArgs.Empty);
415    }
[7248]416    public event EventHandler ProblemChanged { add { } remove { } }
[6934]417    public event EventHandler StoreAlgorithmInEachRunChanged;
[7246]418    private void OnStoreAlgorithmInEachRunChanged() {
[6934]419      EventHandler handler = StoreAlgorithmInEachRunChanged;
420      if (handler != null) handler(this, EventArgs.Empty);
421    }
422    public event EventHandler Prepared;
[7246]423    private void OnPrepared() {
[6934]424      ExecutionState = ExecutionState.Prepared;
425      ExecutionTime = TimeSpan.Zero;
426      foreach (IStatefulItem statefulObject in this.GetObjectGraphObjects().OfType<IStatefulItem>()) {
427        statefulObject.InitializeState();
428      }
429      EventHandler handler = Prepared;
430      if (handler != null) handler(this, EventArgs.Empty);
431    }
432    public event EventHandler Started;
[7246]433    private void OnStarted() {
[6934]434      ExecutionState = ExecutionState.Started;
435      EventHandler handler = Started;
436      if (handler != null) handler(this, EventArgs.Empty);
437    }
438    public event EventHandler Paused;
[7246]439    private void OnPaused() {
[6934]440      ExecutionState = ExecutionState.Paused;
441      EventHandler handler = Paused;
442      if (handler != null) handler(this, EventArgs.Empty);
443    }
444    public event EventHandler Stopped;
[7246]445    private void OnStopped() {
[6934]446      ExecutionState = ExecutionState.Stopped;
447      foreach (IStatefulItem statefulObject in this.GetObjectGraphObjects().OfType<IStatefulItem>()) {
448        statefulObject.ClearState();
449      }
450      runsCounter++;
451      runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
452      EventHandler handler = Stopped;
453      if (handler != null) handler(this, EventArgs.Empty);
454    }
455    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
[7246]456    private void OnExceptionOccurred(Exception exception) {
[6934]457      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
458      if (handler != null) handler(this, new EventArgs<Exception>(exception));
459    }
460
461    public event EventHandler<CancelEventArgs<string>> NameChanging;
[7246]462    private void OnNameChanging(CancelEventArgs<string> e) {
[6934]463      var handler = NameChanging;
464      if (handler != null) handler(this, e);
465    }
466
467    public event EventHandler NameChanged;
[7246]468    private void OnNameChanged() {
[6934]469      var handler = NameChanged;
470      if (handler != null) handler(this, EventArgs.Empty);
471      OnToStringChanged();
472    }
473
474    public event EventHandler DescriptionChanged;
[7246]475    private void OnDescriptionChanged() {
[6934]476      var handler = DescriptionChanged;
477      if (handler != null) handler(this, EventArgs.Empty);
478    }
479
480    public event EventHandler ItemImageChanged;
[7246]481    private void OnItemImageChanged() {
[6934]482      EventHandler handler = ItemImageChanged;
483      if (handler != null) handler(this, EventArgs.Empty);
484    }
485    public event EventHandler ToStringChanged;
[7246]486    private void OnToStringChanged() {
[6934]487      EventHandler handler = ToStringChanged;
488      if (handler != null) handler(this, EventArgs.Empty);
489    }
490
[7246]491    private void DeregisterRunsEvents() {
[6934]492      runs.CollectionReset -= new CollectionItemsChangedEventHandler<IRun>(Runs_CollectionReset);
493    }
[7246]494    private void RegisterRunsEvents() {
[6934]495      runs.CollectionReset += new CollectionItemsChangedEventHandler<IRun>(Runs_CollectionReset);
496    }
[7246]497    private void Runs_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
[6934]498      runsCounter = runs.Count;
499    }
500    #endregion
501  }
502}
Note: See TracBrowser for help on using the repository browser.