source: trunk/sources/HeuristicLab.Algorithms.Benchmarks/3.3/BenchmarkAlgorithm.cs @ 7246

Last change on this file since 7246 was 7246, checked in by swagner, 11 years ago

Changes due to review of benchmark algorithms (#1659):

  • renamed Benchmark to BenchmarkAlgorithm and BenchmarkView to BenchmarkAlgorithmView
  • removed unnecessary assembly references
  • removed IBenchmarkView as the user should be informed that there is no specific view for benchmarks
  • adapted descriptions of plugins and assemblies
  • adapted assembly Guids
  • sealed BenchmarkAlgorithm
  • adapted item names and item descriptions
  • adapted code formatting
File size: 19.9 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2011 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Drawing;
25using System.Linq;
26using System.Threading;
27using System.Threading.Tasks;
28using HeuristicLab.Collections;
29using HeuristicLab.Common;
30using HeuristicLab.Core;
31using HeuristicLab.Data;
32using HeuristicLab.Optimization;
33using HeuristicLab.Parameters;
34using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
35using HeuristicLab.PluginInfrastructure;
36
37namespace HeuristicLab.Algorithms.Benchmarks {
38  [Item("Benchmark Algorithm", "An algorithm to execute performance benchmarks (Linpack, Dhrystone, Whetstone, etc.).")]
39  [Creatable("Algorithms")]
40  [StorableClass]
41  public sealed class BenchmarkAlgorithm : IAlgorithm {
42    private Random random = new Random();
43    private CancellationTokenSource cancellationTokenSource;
44
45    [Storable]
46    private DateTime lastUpdateTime;
47
48    [Storable]
49    private IBenchmark benchmark;
50    public IBenchmark Benchmark {
51      get { return benchmark; }
52      set {
53        if (value == null) throw new ArgumentNullException();
54        benchmark = value;
55      }
56    }
57
58    [Storable]
59    private ExecutionState executionState;
60    public ExecutionState ExecutionState {
61      get { return executionState; }
62      private set {
63        if (executionState != value) {
64          executionState = value;
65          OnExecutionStateChanged();
66          OnItemImageChanged();
67        }
68      }
69    }
70
71    [Storable]
72    private TimeSpan executionTime;
73    public TimeSpan ExecutionTime {
74      get { return executionTime; }
75      private set {
76        executionTime = value;
77        OnExecutionTimeChanged();
78      }
79    }
80
81    [Storable]
82    private bool storeAlgorithmInEachRun;
83    public bool StoreAlgorithmInEachRun {
84      get { return storeAlgorithmInEachRun; }
85      set {
86        if (storeAlgorithmInEachRun != value) {
87          storeAlgorithmInEachRun = value;
88          OnStoreAlgorithmInEachRunChanged();
89        }
90      }
91    }
92
93    [Storable]
94    private int runsCounter;
95
96    [Storable]
97    private RunCollection runs = new RunCollection();
98    public RunCollection Runs {
99      get { return runs; }
100      private set {
101        if (value == null) throw new ArgumentNullException();
102        if (runs != value) {
103          if (runs != null) DeregisterRunsEvents();
104          runs = value;
105          if (runs != null) RegisterRunsEvents();
106        }
107      }
108    }
109
110    [Storable]
111    private ResultCollection results;
112    public ResultCollection Results {
113      get { return results; }
114    }
115
116    public Type ProblemType {
117      get { return typeof(IProblem); }
118    }
119
120    [Storable]
121    private IProblem problem;
122    public IProblem Problem {
123      get { return problem; }
124      set {
125        if (problem != value) {
126          if ((value != null) && !ProblemType.IsInstanceOfType(value)) throw new ArgumentException("Invalid problem type.");
127          if (problem != null) DeregisterProblemEvents();
128          problem = value;
129          if (problem != null) RegisterProblemEvents();
130          OnProblemChanged();
131          Prepare();
132        }
133      }
134    }
135
136    [Storable]
137    private string name;
138    public string Name {
139      get { return name; }
140      set {
141        if (!CanChangeName) throw new NotSupportedException("Name cannot be changed.");
142        if (!(name.Equals(value) || (value == null) && (name == string.Empty))) {
143          CancelEventArgs<string> e = value == null ? new CancelEventArgs<string>(string.Empty) : new CancelEventArgs<string>(value);
144          OnNameChanging(e);
145          if (!e.Cancel) {
146            name = value == null ? string.Empty : value;
147            OnNameChanged();
148          }
149        }
150      }
151    }
152
153    public bool CanChangeName {
154      get { return false; }
155    }
156
157    [Storable]
158    private string description;
159    public string Description {
160      get { return description; }
161      set {
162        if (!CanChangeDescription) throw new NotSupportedException("Description cannot be changed.");
163        if (!(description.Equals(value) || (value == null) && (description == string.Empty))) {
164          description = value == null ? string.Empty : value;
165          OnDescriptionChanged();
166        }
167      }
168    }
169
170    public bool CanChangeDescription {
171      get { return false; }
172    }
173
174    public string ItemName {
175      get { return ItemAttribute.GetName(this.GetType()); }
176    }
177
178    public string ItemDescription {
179      get { return ItemAttribute.GetDescription(this.GetType()); }
180    }
181
182    public Version ItemVersion {
183      get { return ItemAttribute.GetVersion(this.GetType()); }
184    }
185
186    public static Image StaticItemImage {
187      get { return HeuristicLab.Common.Resources.VSImageLibrary.Event; }
188    }
189    public Image ItemImage {
190      get {
191        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
192        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
193        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
194        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
195        else return ItemAttribute.GetImage(this.GetType());
196      }
197    }
198
199    [Storable]
200    private ParameterCollection parameters = new ParameterCollection();
201
202    public IKeyedItemCollection<string, IParameter> Parameters {
203      get { return parameters; }
204    }
205
206    private ReadOnlyKeyedItemCollection<string, IParameter> readOnlyParameters;
207
208    IKeyedItemCollection<string, IParameter> IParameterizedItem.Parameters {
209      get {
210        if (readOnlyParameters == null) readOnlyParameters = parameters.AsReadOnly();
211        return readOnlyParameters;
212      }
213    }
214
215    public IEnumerable<IOptimizer> NestedOptimizers {
216      get { return Enumerable.Empty<IOptimizer>(); }
217    }
218
219    #region Parameter Properties
220
221    public ConstrainedValueParameter<IBenchmark> BenchmarkParameter {
222      get { return (ConstrainedValueParameter<IBenchmark>)Parameters["Benchmark"]; }
223    }
224
225    private ValueParameter<IntValue> ChunkSizeParameter {
226      get { return (ValueParameter<IntValue>)Parameters["ChunkSize"]; }
227    }
228
229    private ValueParameter<DoubleValue> TimeLimitParameter {
230      get { return (ValueParameter<DoubleValue>)Parameters["TimeLimit"]; }
231    }
232
233    #endregion
234
235    #region Constructors
236
237    [StorableConstructor]
238    public BenchmarkAlgorithm(bool deserializing) { }
239
240    public BenchmarkAlgorithm() {
241      name = ItemName;
242      description = ItemDescription;
243      parameters = new ParameterCollection();
244      readOnlyParameters = null;
245      executionState = ExecutionState.Stopped;
246      executionTime = TimeSpan.Zero;
247      storeAlgorithmInEachRun = false;
248      runsCounter = 0;
249      Runs = new RunCollection();
250      results = new ResultCollection();
251      CreateParameters();
252      DiscoverBenchmarks();
253      Prepare();
254    }
255
256    public BenchmarkAlgorithm(BenchmarkAlgorithm original, Cloner cloner) {
257      cloner.RegisterClonedObject(original, this);
258      name = original.name;
259      description = original.description;
260      parameters = cloner.Clone(original.parameters);
261      readOnlyParameters = null;
262      if (ExecutionState == ExecutionState.Started) throw new InvalidOperationException(string.Format("Clone not allowed in execution state \"{0}\".", ExecutionState));
263      executionState = original.executionState;
264      executionTime = original.executionTime;
265      storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
266      runsCounter = original.runsCounter;
267      runs = cloner.Clone(original.runs);
268      Initialize();
269
270      results = cloner.Clone(original.results);
271      DiscoverBenchmarks();
272      Prepare();
273    }
274
275    #endregion
276
277    private void CreateParameters() {
278      Parameters.Add(new ValueParameter<IntValue>("ChunkSize", "The size (MB) of the chunk array that gets generated", new IntValue(0)));
279      Parameters.Add(new ValueParameter<DoubleValue>("TimeLimit", "The time limit (in minutes) for a benchmark run. Zero means a fixed number of iterations", new DoubleValue(0)));
280    }
281
282    private void DiscoverBenchmarks() {
283      var benchmarks = from t in ApplicationManager.Manager.GetTypes(typeof(IBenchmark))
284                       select t;
285      ItemSet<IBenchmark> values = new ItemSet<IBenchmark>();
286      foreach (var benchmark in benchmarks) {
287        IBenchmark b = (IBenchmark)Activator.CreateInstance(benchmark);
288        values.Add(b);
289      }
290      string paramName = "Benchmark";
291      if (!Parameters.ContainsKey(paramName)) {
292        if (values.Count > 0) {
293          Parameters.Add(new ConstrainedValueParameter<IBenchmark>(paramName, values, values.First(a => a is IBenchmark)));
294        } else {
295          Parameters.Add(new ConstrainedValueParameter<IBenchmark>(paramName, values));
296        }
297      }
298    }
299
300    private void Initialize() {
301      if (problem != null) RegisterProblemEvents();
302      if (runs != null) RegisterRunsEvents();
303    }
304
305    public void Prepare() {
306      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused) && (ExecutionState != ExecutionState.Stopped))
307        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
308      results.Clear();
309      OnPrepared();
310    }
311
312    public void Prepare(bool clearRuns) {
313      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused) && (ExecutionState != ExecutionState.Stopped))
314        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
315      if (clearRuns) runs.Clear();
316      Prepare();
317    }
318
319    public void Pause() {
320      if (ExecutionState != ExecutionState.Started)
321        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
322    }
323    public void Stop() {
324      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
325        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".", ExecutionState));
326      cancellationTokenSource.Cancel();
327    }
328
329    public void Start() {
330      cancellationTokenSource = new CancellationTokenSource();
331      OnStarted();
332      Task task = Task.Factory.StartNew(Run, cancellationTokenSource.Token, cancellationTokenSource.Token);
333      task.ContinueWith(t => {
334        try {
335          t.Wait();
336        }
337        catch (AggregateException ex) {
338          try {
339            ex.Flatten().Handle(x => x is OperationCanceledException);
340          }
341          catch (AggregateException remaining) {
342            if (remaining.InnerExceptions.Count == 1) OnExceptionOccurred(remaining.InnerExceptions[0]);
343            else OnExceptionOccurred(remaining);
344          }
345        }
346
347        cancellationTokenSource.Dispose();
348        cancellationTokenSource = null;
349        OnStopped();
350      });
351    }
352
353    private void Run(object state) {
354      CancellationToken cancellationToken = (CancellationToken)state;
355      lastUpdateTime = DateTime.Now;
356      System.Timers.Timer timer = new System.Timers.Timer(250);
357      timer.AutoReset = true;
358      timer.Elapsed += new System.Timers.ElapsedEventHandler(timer_Elapsed);
359      timer.Start();
360      try {
361        Benchmark = (IBenchmark)BenchmarkParameter.ActualValue;
362        int chunkSize = ((IntValue)ChunkSizeParameter.ActualValue).Value;
363        if (chunkSize > 0) {
364          Benchmark.ChunkData = CreateDataChuck(chunkSize);
365        } else if (chunkSize < 0) {
366          throw new ArgumentException("ChunkSize must not be negativ.");
367        }
368        TimeSpan timelimit = TimeSpan.FromMinutes(((DoubleValue)TimeLimitParameter.ActualValue).Value);
369        if (timelimit.TotalMilliseconds < 0) {
370          throw new ArgumentException("TimeLimit must not be negativ. ");
371        }
372        Benchmark.TimeLimit = timelimit;
373        Benchmark.Run(cancellationToken, results);
374      }
375      catch (OperationCanceledException) {
376      }
377      finally {
378        timer.Elapsed -= new System.Timers.ElapsedEventHandler(timer_Elapsed);
379        timer.Stop();
380        ExecutionTime += DateTime.Now - lastUpdateTime;
381      }
382    }
383
384    private void timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) {
385      System.Timers.Timer timer = (System.Timers.Timer)sender;
386      timer.Enabled = false;
387      DateTime now = DateTime.Now;
388      ExecutionTime += now - lastUpdateTime;
389      lastUpdateTime = now;
390      timer.Enabled = true;
391    }
392
393    public void CollectResultValues(IDictionary<string, IItem> values) {
394      values.Add("Execution Time", new TimeSpanValue(ExecutionTime));
395      CollectResultsRecursively("", Results, values);
396    }
397
398    private void CollectResultsRecursively(string path, ResultCollection results, IDictionary<string, IItem> values) {
399      foreach (IResult result in results) {
400        values.Add(path + result.Name, result.Value);
401        ResultCollection childCollection = result.Value as ResultCollection;
402        if (childCollection != null) {
403          CollectResultsRecursively(path + result.Name + ".", childCollection, values);
404        }
405      }
406    }
407
408    public void CollectParameterValues(IDictionary<string, IItem> values) {
409      foreach (IValueParameter param in parameters.OfType<IValueParameter>()) {
410        if (param.GetsCollected && param.Value != null) values.Add(param.Name, param.Value);
411        if (param.Value is IParameterizedItem) {
412          Dictionary<string, IItem> children = new Dictionary<string, IItem>();
413          ((IParameterizedItem)param.Value).CollectParameterValues(children);
414          foreach (string key in children.Keys)
415            values.Add(param.Name + "." + key, children[key]);
416        }
417      }
418    }
419
420    private byte[][] CreateDataChuck(int megaBytes) {
421      if (megaBytes <= 0) {
422        throw new ArgumentException("MegaBytes must be greater than zero", "megaBytes");
423      }
424      byte[][] chunk = new byte[megaBytes][];
425      for (int i = 0; i < chunk.Length; i++) {
426        chunk[i] = new byte[1024 * 1024];
427        random.NextBytes(chunk[i]);
428      }
429      return chunk;
430    }
431
432    #region Events
433
434    public event EventHandler ExecutionStateChanged;
435    private void OnExecutionStateChanged() {
436      EventHandler handler = ExecutionStateChanged;
437      if (handler != null) handler(this, EventArgs.Empty);
438    }
439    public event EventHandler ExecutionTimeChanged;
440    private void OnExecutionTimeChanged() {
441      EventHandler handler = ExecutionTimeChanged;
442      if (handler != null) handler(this, EventArgs.Empty);
443    }
444    public event EventHandler ProblemChanged;
445    private void OnProblemChanged() {
446      EventHandler handler = ProblemChanged;
447      if (handler != null) handler(this, EventArgs.Empty);
448    }
449    public event EventHandler StoreAlgorithmInEachRunChanged;
450    private void OnStoreAlgorithmInEachRunChanged() {
451      EventHandler handler = StoreAlgorithmInEachRunChanged;
452      if (handler != null) handler(this, EventArgs.Empty);
453    }
454    public event EventHandler Prepared;
455    private void OnPrepared() {
456      ExecutionState = ExecutionState.Prepared;
457      ExecutionTime = TimeSpan.Zero;
458      foreach (IStatefulItem statefulObject in this.GetObjectGraphObjects().OfType<IStatefulItem>()) {
459        statefulObject.InitializeState();
460      }
461      EventHandler handler = Prepared;
462      if (handler != null) handler(this, EventArgs.Empty);
463    }
464    public event EventHandler Started;
465    private void OnStarted() {
466      ExecutionState = ExecutionState.Started;
467      EventHandler handler = Started;
468      if (handler != null) handler(this, EventArgs.Empty);
469    }
470    public event EventHandler Paused;
471    private void OnPaused() {
472      ExecutionState = ExecutionState.Paused;
473      EventHandler handler = Paused;
474      if (handler != null) handler(this, EventArgs.Empty);
475    }
476    public event EventHandler Stopped;
477    private void OnStopped() {
478      ExecutionState = ExecutionState.Stopped;
479      foreach (IStatefulItem statefulObject in this.GetObjectGraphObjects().OfType<IStatefulItem>()) {
480        statefulObject.ClearState();
481      }
482      runsCounter++;
483      runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
484      EventHandler handler = Stopped;
485      if (handler != null) handler(this, EventArgs.Empty);
486    }
487    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
488    private void OnExceptionOccurred(Exception exception) {
489      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
490      if (handler != null) handler(this, new EventArgs<Exception>(exception));
491    }
492
493    public event EventHandler<CancelEventArgs<string>> NameChanging;
494    private void OnNameChanging(CancelEventArgs<string> e) {
495      var handler = NameChanging;
496      if (handler != null) handler(this, e);
497    }
498
499    public event EventHandler NameChanged;
500    private void OnNameChanged() {
501      var handler = NameChanged;
502      if (handler != null) handler(this, EventArgs.Empty);
503      OnToStringChanged();
504    }
505
506    public event EventHandler DescriptionChanged;
507    private void OnDescriptionChanged() {
508      var handler = DescriptionChanged;
509      if (handler != null) handler(this, EventArgs.Empty);
510    }
511
512    public event EventHandler ItemImageChanged;
513    private void OnItemImageChanged() {
514      EventHandler handler = ItemImageChanged;
515      if (handler != null) handler(this, EventArgs.Empty);
516    }
517    public event EventHandler ToStringChanged;
518    private void OnToStringChanged() {
519      EventHandler handler = ToStringChanged;
520      if (handler != null) handler(this, EventArgs.Empty);
521    }
522
523    private void DeregisterProblemEvents() {
524      problem.OperatorsChanged -= new EventHandler(Problem_OperatorsChanged);
525      problem.Reset -= new EventHandler(Problem_Reset);
526    }
527    private void RegisterProblemEvents() {
528      problem.OperatorsChanged += new EventHandler(Problem_OperatorsChanged);
529      problem.Reset += new EventHandler(Problem_Reset);
530    }
531    private void Problem_OperatorsChanged(object sender, EventArgs e) { }
532    private void Problem_Reset(object sender, EventArgs e) {
533      Prepare();
534    }
535
536    private void DeregisterRunsEvents() {
537      runs.CollectionReset -= new CollectionItemsChangedEventHandler<IRun>(Runs_CollectionReset);
538    }
539    private void RegisterRunsEvents() {
540      runs.CollectionReset += new CollectionItemsChangedEventHandler<IRun>(Runs_CollectionReset);
541    }
542    private void Runs_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
543      runsCounter = runs.Count;
544    }
545
546    #endregion
547
548    #region Clone
549
550    public IDeepCloneable Clone(Cloner cloner) {
551      return new BenchmarkAlgorithm(this, cloner);
552    }
553
554    public object Clone() {
555      return Clone(new Cloner());
556    }
557
558    #endregion
559  }
560}
Note: See TracBrowser for help on using the repository browser.