source: trunk/sources/HeuristicLab.Algorithms.Benchmarks/3.3/BenchmarkAlgorithm.cs @ 7248

Last change on this file since 7248 was 7248, checked in by swagner, 11 years ago

Changes due to review of benchmark algorithms (#1659):

  • removed unnecessary problem-specific parts in BenchmarkAlgorithm
  • enabled name and description changes in BenchmarkAlgorithm
  • adapted parameter descriptions
  • implemented ToString in BenchmarkAlgorithm
  • renamed CreateDataChuck to CreateChunkData
  • adapted code formatting
File size: 18.9 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2011 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Drawing;
25using System.Linq;
26using System.Threading;
27using System.Threading.Tasks;
28using HeuristicLab.Collections;
29using HeuristicLab.Common;
30using HeuristicLab.Core;
31using HeuristicLab.Data;
32using HeuristicLab.Optimization;
33using HeuristicLab.Parameters;
34using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
35using HeuristicLab.PluginInfrastructure;
36
37namespace HeuristicLab.Algorithms.Benchmarks {
38  [Item("Benchmark Algorithm", "An algorithm to execute performance benchmarks (Linpack, Dhrystone, Whetstone, etc.).")]
39  [Creatable("Algorithms")]
40  [StorableClass]
41  public sealed class BenchmarkAlgorithm : IAlgorithm {
42    private CancellationTokenSource cancellationTokenSource;
43
44    public string ItemName {
45      get { return ItemAttribute.GetName(this.GetType()); }
46    }
47    public string ItemDescription {
48      get { return ItemAttribute.GetDescription(this.GetType()); }
49    }
50    public Version ItemVersion {
51      get { return ItemAttribute.GetVersion(this.GetType()); }
52    }
53    public static Image StaticItemImage {
54      get { return HeuristicLab.Common.Resources.VSImageLibrary.Event; }
55    }
56    public Image ItemImage {
57      get {
58        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
59        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
60        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
61        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
62        else return ItemAttribute.GetImage(this.GetType());
63      }
64    }
65
66    [Storable]
67    private DateTime lastUpdateTime;
68
69    [Storable]
70    private IBenchmark benchmark;
71    public IBenchmark Benchmark {
72      get { return benchmark; }
73      set {
74        if (value == null) throw new ArgumentNullException();
75        benchmark = value;
76      }
77    }
78
79    [Storable]
80    private ExecutionState executionState;
81    public ExecutionState ExecutionState {
82      get { return executionState; }
83      private set {
84        if (executionState != value) {
85          executionState = value;
86          OnExecutionStateChanged();
87          OnItemImageChanged();
88        }
89      }
90    }
91
92    [Storable]
93    private TimeSpan executionTime;
94    public TimeSpan ExecutionTime {
95      get { return executionTime; }
96      private set {
97        executionTime = value;
98        OnExecutionTimeChanged();
99      }
100    }
101
102    [Storable]
103    private bool storeAlgorithmInEachRun;
104    public bool StoreAlgorithmInEachRun {
105      get { return storeAlgorithmInEachRun; }
106      set {
107        if (storeAlgorithmInEachRun != value) {
108          storeAlgorithmInEachRun = value;
109          OnStoreAlgorithmInEachRunChanged();
110        }
111      }
112    }
113
114    [Storable]
115    private int runsCounter;
116
117    [Storable]
118    private RunCollection runs = new RunCollection();
119    public RunCollection Runs {
120      get { return runs; }
121      private set {
122        if (value == null) throw new ArgumentNullException();
123        if (runs != value) {
124          if (runs != null) DeregisterRunsEvents();
125          runs = value;
126          if (runs != null) RegisterRunsEvents();
127        }
128      }
129    }
130
131    [Storable]
132    private ResultCollection results;
133    public ResultCollection Results {
134      get { return results; }
135    }
136
137    public Type ProblemType {
138      get {
139        // BenchmarkAlgorithm does not have a problem, so return a type which is no problem for sure
140        return typeof(BenchmarkAlgorithm);
141      }
142    }
143
144    public IProblem Problem {
145      get { return null; }
146      set { throw new NotImplementedException("BenchmarkAlgorithm does not have a problem."); }
147    }
148
149    [Storable]
150    private string name;
151    public string Name {
152      get { return name; }
153      set {
154        if (!CanChangeName) throw new NotSupportedException("Name cannot be changed.");
155        if (!(name.Equals(value) || (value == null) && (name == string.Empty))) {
156          CancelEventArgs<string> e = value == null ? new CancelEventArgs<string>(string.Empty) : new CancelEventArgs<string>(value);
157          OnNameChanging(e);
158          if (!e.Cancel) {
159            name = value == null ? string.Empty : value;
160            OnNameChanged();
161          }
162        }
163      }
164    }
165    public bool CanChangeName {
166      get { return true; }
167    }
168
169    [Storable]
170    private string description;
171    public string Description {
172      get { return description; }
173      set {
174        if (!CanChangeDescription) throw new NotSupportedException("Description cannot be changed.");
175        if (!(description.Equals(value) || (value == null) && (description == string.Empty))) {
176          description = value == null ? string.Empty : value;
177          OnDescriptionChanged();
178        }
179      }
180    }
181    public bool CanChangeDescription {
182      get { return true; }
183    }
184
185    [Storable]
186    private ParameterCollection parameters = new ParameterCollection();
187    public IKeyedItemCollection<string, IParameter> Parameters {
188      get { return parameters; }
189    }
190    private ReadOnlyKeyedItemCollection<string, IParameter> readOnlyParameters;
191    IKeyedItemCollection<string, IParameter> IParameterizedItem.Parameters {
192      get {
193        if (readOnlyParameters == null) readOnlyParameters = parameters.AsReadOnly();
194        return readOnlyParameters;
195      }
196    }
197
198    public IEnumerable<IOptimizer> NestedOptimizers {
199      get { return Enumerable.Empty<IOptimizer>(); }
200    }
201
202    #region Parameter Properties
203    public ConstrainedValueParameter<IBenchmark> BenchmarkParameter {
204      get { return (ConstrainedValueParameter<IBenchmark>)Parameters["Benchmark"]; }
205    }
206    private ValueParameter<IntValue> ChunkSizeParameter {
207      get { return (ValueParameter<IntValue>)Parameters["ChunkSize"]; }
208    }
209    private ValueParameter<DoubleValue> TimeLimitParameter {
210      get { return (ValueParameter<DoubleValue>)Parameters["TimeLimit"]; }
211    }
212    #endregion
213
214    #region Constructors
215    [StorableConstructor]
216    private BenchmarkAlgorithm(bool deserializing) { }
217    private BenchmarkAlgorithm(BenchmarkAlgorithm original, Cloner cloner) {
218      if (original.ExecutionState == ExecutionState.Started) throw new InvalidOperationException(string.Format("Clone not allowed in execution state \"{0}\".", ExecutionState));
219      cloner.RegisterClonedObject(original, this);
220      name = original.name;
221      description = original.description;
222      parameters = cloner.Clone(original.parameters);
223      readOnlyParameters = null;
224      executionState = original.executionState;
225      executionTime = original.executionTime;
226      storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
227      runsCounter = original.runsCounter;
228      Runs = cloner.Clone(original.runs);
229      results = cloner.Clone(original.results);
230    }
231    public BenchmarkAlgorithm() {
232      name = ItemName;
233      description = ItemDescription;
234      parameters = new ParameterCollection();
235      readOnlyParameters = null;
236      executionState = ExecutionState.Stopped;
237      executionTime = TimeSpan.Zero;
238      storeAlgorithmInEachRun = false;
239      runsCounter = 0;
240      Runs = new RunCollection();
241      results = new ResultCollection();
242      CreateParameters();
243      DiscoverBenchmarks();
244      Prepare();
245    }
246    #endregion
247
248    private void CreateParameters() {
249      Parameters.Add(new ValueParameter<IntValue>("ChunkSize", "The size in MB of the chunk data array that is generated.", new IntValue(0)));
250      Parameters.Add(new ValueParameter<DoubleValue>("TimeLimit", "The time limit in minutes for a benchmark run (zero means a fixed number of iterations).", new DoubleValue(0)));
251    }
252    private void DiscoverBenchmarks() {
253      var benchmarks = from t in ApplicationManager.Manager.GetTypes(typeof(IBenchmark))
254                       select t;
255      ItemSet<IBenchmark> values = new ItemSet<IBenchmark>();
256      foreach (var benchmark in benchmarks) {
257        IBenchmark b = (IBenchmark)Activator.CreateInstance(benchmark);
258        values.Add(b);
259      }
260      string paramName = "Benchmark";
261      if (!Parameters.ContainsKey(paramName)) {
262        if (values.Count > 0) {
263          Parameters.Add(new ConstrainedValueParameter<IBenchmark>(paramName, "The benchmark which should be executed.", values, values.First(a => a is IBenchmark)));
264        } else {
265          Parameters.Add(new ConstrainedValueParameter<IBenchmark>(paramName, "The benchmark which should be executed.", values));
266        }
267      }
268    }
269
270    public IDeepCloneable Clone(Cloner cloner) {
271      return new BenchmarkAlgorithm(this, cloner);
272    }
273    public object Clone() {
274      return Clone(new Cloner());
275    }
276
277    public override string ToString() {
278      return Name;
279    }
280
281    public void Prepare() {
282      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused) && (ExecutionState != ExecutionState.Stopped))
283        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
284      results.Clear();
285      OnPrepared();
286    }
287    public void Prepare(bool clearRuns) {
288      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused) && (ExecutionState != ExecutionState.Stopped))
289        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
290      if (clearRuns) runs.Clear();
291      Prepare();
292    }
293    public void Pause() {
294      if (ExecutionState != ExecutionState.Started)
295        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
296    }
297    public void Stop() {
298      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
299        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".", ExecutionState));
300      cancellationTokenSource.Cancel();
301    }
302    public void Start() {
303      cancellationTokenSource = new CancellationTokenSource();
304      OnStarted();
305      Task task = Task.Factory.StartNew(Run, cancellationTokenSource.Token, cancellationTokenSource.Token);
306      task.ContinueWith(t => {
307        try {
308          t.Wait();
309        }
310        catch (AggregateException ex) {
311          try {
312            ex.Flatten().Handle(x => x is OperationCanceledException);
313          }
314          catch (AggregateException remaining) {
315            if (remaining.InnerExceptions.Count == 1) OnExceptionOccurred(remaining.InnerExceptions[0]);
316            else OnExceptionOccurred(remaining);
317          }
318        }
319
320        cancellationTokenSource.Dispose();
321        cancellationTokenSource = null;
322        OnStopped();
323      });
324    }
325
326    private void Run(object state) {
327      CancellationToken cancellationToken = (CancellationToken)state;
328      lastUpdateTime = DateTime.Now;
329      System.Timers.Timer timer = new System.Timers.Timer(250);
330      timer.AutoReset = true;
331      timer.Elapsed += new System.Timers.ElapsedEventHandler(timer_Elapsed);
332      timer.Start();
333      try {
334        Benchmark = (IBenchmark)BenchmarkParameter.ActualValue;
335        int chunkSize = ((IntValue)ChunkSizeParameter.ActualValue).Value;
336        if (chunkSize > 0) {
337          Benchmark.ChunkData = CreateChunkData(chunkSize);
338        } else if (chunkSize < 0) {
339          throw new ArgumentException("ChunkSize must not be negativ.");
340        }
341        TimeSpan timelimit = TimeSpan.FromMinutes(((DoubleValue)TimeLimitParameter.ActualValue).Value);
342        if (timelimit.TotalMilliseconds < 0) {
343          throw new ArgumentException("TimeLimit must not be negativ. ");
344        }
345        Benchmark.TimeLimit = timelimit;
346        Benchmark.Run(cancellationToken, results);
347      }
348      catch (OperationCanceledException) {
349      }
350      finally {
351        timer.Elapsed -= new System.Timers.ElapsedEventHandler(timer_Elapsed);
352        timer.Stop();
353        ExecutionTime += DateTime.Now - lastUpdateTime;
354      }
355    }
356
357    private void timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) {
358      System.Timers.Timer timer = (System.Timers.Timer)sender;
359      timer.Enabled = false;
360      DateTime now = DateTime.Now;
361      ExecutionTime += now - lastUpdateTime;
362      lastUpdateTime = now;
363      timer.Enabled = true;
364    }
365
366    public void CollectResultValues(IDictionary<string, IItem> values) {
367      values.Add("Execution Time", new TimeSpanValue(ExecutionTime));
368      CollectResultsRecursively("", Results, values);
369    }
370    private void CollectResultsRecursively(string path, ResultCollection results, IDictionary<string, IItem> values) {
371      foreach (IResult result in results) {
372        values.Add(path + result.Name, result.Value);
373        ResultCollection childCollection = result.Value as ResultCollection;
374        if (childCollection != null) {
375          CollectResultsRecursively(path + result.Name + ".", childCollection, values);
376        }
377      }
378    }
379    public void CollectParameterValues(IDictionary<string, IItem> values) {
380      foreach (IValueParameter param in parameters.OfType<IValueParameter>()) {
381        if (param.GetsCollected && param.Value != null) values.Add(param.Name, param.Value);
382        if (param.Value is IParameterizedItem) {
383          Dictionary<string, IItem> children = new Dictionary<string, IItem>();
384          ((IParameterizedItem)param.Value).CollectParameterValues(children);
385          foreach (string key in children.Keys)
386            values.Add(param.Name + "." + key, children[key]);
387        }
388      }
389    }
390
391    private byte[][] CreateChunkData(int megaBytes) {
392      if (megaBytes <= 0) {
393        throw new ArgumentException("MegaBytes must be greater than zero", "megaBytes");
394      }
395      Random random = new Random();
396      byte[][] chunk = new byte[megaBytes][];
397      for (int i = 0; i < chunk.Length; i++) {
398        chunk[i] = new byte[1024 * 1024];
399        random.NextBytes(chunk[i]);
400      }
401      return chunk;
402    }
403
404    #region Events
405    public event EventHandler ExecutionStateChanged;
406    private void OnExecutionStateChanged() {
407      EventHandler handler = ExecutionStateChanged;
408      if (handler != null) handler(this, EventArgs.Empty);
409    }
410    public event EventHandler ExecutionTimeChanged;
411    private void OnExecutionTimeChanged() {
412      EventHandler handler = ExecutionTimeChanged;
413      if (handler != null) handler(this, EventArgs.Empty);
414    }
415    public event EventHandler ProblemChanged { add { } remove { } }
416    public event EventHandler StoreAlgorithmInEachRunChanged;
417    private void OnStoreAlgorithmInEachRunChanged() {
418      EventHandler handler = StoreAlgorithmInEachRunChanged;
419      if (handler != null) handler(this, EventArgs.Empty);
420    }
421    public event EventHandler Prepared;
422    private void OnPrepared() {
423      ExecutionState = ExecutionState.Prepared;
424      ExecutionTime = TimeSpan.Zero;
425      foreach (IStatefulItem statefulObject in this.GetObjectGraphObjects().OfType<IStatefulItem>()) {
426        statefulObject.InitializeState();
427      }
428      EventHandler handler = Prepared;
429      if (handler != null) handler(this, EventArgs.Empty);
430    }
431    public event EventHandler Started;
432    private void OnStarted() {
433      ExecutionState = ExecutionState.Started;
434      EventHandler handler = Started;
435      if (handler != null) handler(this, EventArgs.Empty);
436    }
437    public event EventHandler Paused;
438    private void OnPaused() {
439      ExecutionState = ExecutionState.Paused;
440      EventHandler handler = Paused;
441      if (handler != null) handler(this, EventArgs.Empty);
442    }
443    public event EventHandler Stopped;
444    private void OnStopped() {
445      ExecutionState = ExecutionState.Stopped;
446      foreach (IStatefulItem statefulObject in this.GetObjectGraphObjects().OfType<IStatefulItem>()) {
447        statefulObject.ClearState();
448      }
449      runsCounter++;
450      runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
451      EventHandler handler = Stopped;
452      if (handler != null) handler(this, EventArgs.Empty);
453    }
454    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
455    private void OnExceptionOccurred(Exception exception) {
456      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
457      if (handler != null) handler(this, new EventArgs<Exception>(exception));
458    }
459
460    public event EventHandler<CancelEventArgs<string>> NameChanging;
461    private void OnNameChanging(CancelEventArgs<string> e) {
462      var handler = NameChanging;
463      if (handler != null) handler(this, e);
464    }
465
466    public event EventHandler NameChanged;
467    private void OnNameChanged() {
468      var handler = NameChanged;
469      if (handler != null) handler(this, EventArgs.Empty);
470      OnToStringChanged();
471    }
472
473    public event EventHandler DescriptionChanged;
474    private void OnDescriptionChanged() {
475      var handler = DescriptionChanged;
476      if (handler != null) handler(this, EventArgs.Empty);
477    }
478
479    public event EventHandler ItemImageChanged;
480    private void OnItemImageChanged() {
481      EventHandler handler = ItemImageChanged;
482      if (handler != null) handler(this, EventArgs.Empty);
483    }
484    public event EventHandler ToStringChanged;
485    private void OnToStringChanged() {
486      EventHandler handler = ToStringChanged;
487      if (handler != null) handler(this, EventArgs.Empty);
488    }
489
490    private void DeregisterRunsEvents() {
491      runs.CollectionReset -= new CollectionItemsChangedEventHandler<IRun>(Runs_CollectionReset);
492    }
493    private void RegisterRunsEvents() {
494      runs.CollectionReset += new CollectionItemsChangedEventHandler<IRun>(Runs_CollectionReset);
495    }
496    private void Runs_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
497      runsCounter = runs.Count;
498    }
499    #endregion
500  }
501}
Note: See TracBrowser for help on using the repository browser.