source: branches/Benchmarking/sources/HeuristicLab.Algorithms.Benchmarks/3.3/Benchmark.cs @ 6948

Last change on this file since 6948 was 6948, checked in by spimming, 11 years ago

#1659:

  • restructuring of the benchmarking algorithms
  • common interface for benchmarks
  • 'benchmark' class discovers benchmarking algorithms
  • added 'TimeLimit' and 'ChunkSize' parameters to benchmark
File size: 20.3 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2011 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Drawing;
25using System.Linq;
26using System.Threading;
27using System.Threading.Tasks;
28using HeuristicLab.Collections;
29using HeuristicLab.Common;
30using HeuristicLab.Core;
31using HeuristicLab.Data;
32using HeuristicLab.Optimization;
33using HeuristicLab.Parameters;
34using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
35using HeuristicLab.PluginInfrastructure;
36
37namespace HeuristicLab.Algorithms.Benchmarks {
38  /// <summary>
39  /// A base class for benchmarks.
40  /// </summary>
41  [Item("Benchmark", "A wrapper for benchmark algorithms.")]
42  [Creatable("Benchmarks")]
43  [StorableClass]
44  public class Benchmark : IAlgorithm {
45    private Random random = new Random();
46
47    [Storable]
48    private DateTime lastUpdateTime;
49
50    [Storable]
51    private IBenchmark benchmarkAlgorithm;
52    public IBenchmark BenchmarkAlgorithm {
53      get { return benchmarkAlgorithm; }
54      set {
55        if (value == null) throw new ArgumentNullException();
56        benchmarkAlgorithm = value;
57      }
58    }
59
60    private CancellationTokenSource cancellationTokenSource;
61
62    [Storable]
63    private ExecutionState executionState;
64    public ExecutionState ExecutionState {
65      get { return executionState; }
66      private set {
67        if (executionState != value) {
68          executionState = value;
69          OnExecutionStateChanged();
70          OnItemImageChanged();
71        }
72      }
73    }
74
75    [Storable]
76    private TimeSpan executionTime;
77    public TimeSpan ExecutionTime {
78      get { return executionTime; }
79      protected set {
80        executionTime = value;
81        OnExecutionTimeChanged();
82      }
83    }
84
85    [Storable]
86    private bool storeAlgorithmInEachRun;
87    public bool StoreAlgorithmInEachRun {
88      get { return storeAlgorithmInEachRun; }
89      set {
90        if (storeAlgorithmInEachRun != value) {
91          storeAlgorithmInEachRun = value;
92          OnStoreAlgorithmInEachRunChanged();
93        }
94      }
95    }
96
97    [Storable]
98    protected int runsCounter;
99
100    [Storable]
101    private RunCollection runs = new RunCollection();
102    public RunCollection Runs {
103      get { return runs; }
104      protected set {
105        if (value == null) throw new ArgumentNullException();
106        if (runs != value) {
107          if (runs != null) DeregisterRunsEvents();
108          runs = value;
109          if (runs != null) RegisterRunsEvents();
110        }
111      }
112    }
113
114    [Storable]
115    private ResultCollection results;
116    public ResultCollection Results {
117      get { return results; }
118    }
119
120    [Storable]
121    private IProblem problem;
122    public IProblem Problem {
123      get { return problem; }
124      set {
125        if (problem != value) {
126          if ((value != null) && !ProblemType.IsInstanceOfType(value)) throw new ArgumentException("Invalid problem type.");
127          if (problem != null) DeregisterProblemEvents();
128          problem = value;
129          if (problem != null) RegisterProblemEvents();
130          OnProblemChanged();
131          Prepare();
132        }
133      }
134    }
135
136    public Type ProblemType {
137      get { return typeof(IProblem); }
138    }
139
140    [Storable]
141    protected string name;
142
143    public string Name {
144      get { return name; }
145      set {
146        if (!CanChangeName) throw new NotSupportedException("Name cannot be changed.");
147        if (!(name.Equals(value) || (value == null) && (name == string.Empty))) {
148          CancelEventArgs<string> e = value == null ? new CancelEventArgs<string>(string.Empty) : new CancelEventArgs<string>(value);
149          OnNameChanging(e);
150          if (!e.Cancel) {
151            name = value == null ? string.Empty : value;
152            OnNameChanged();
153          }
154        }
155      }
156    }
157
158    public bool CanChangeName {
159      get { return false; }
160    }
161
162    [Storable]
163    protected string description;
164    public string Description {
165      get { return description; }
166      set {
167        if (!CanChangeDescription) throw new NotSupportedException("Description cannot be changed.");
168        if (!(description.Equals(value) || (value == null) && (description == string.Empty))) {
169          description = value == null ? string.Empty : value;
170          OnDescriptionChanged();
171        }
172      }
173    }
174
175    public bool CanChangeDescription {
176      get { return false; }
177    }
178
179    public string ItemName {
180      get { return ItemAttribute.GetName(this.GetType()); }
181    }
182
183    public string ItemDescription {
184      get { return ItemAttribute.GetDescription(this.GetType()); }
185    }
186
187    public Version ItemVersion {
188      get { return ItemAttribute.GetVersion(this.GetType()); }
189    }
190
191    public Image ItemImage {
192      get {
193        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
194        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
195        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
196        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
197        else return HeuristicLab.Common.Resources.VSImageLibrary.Event;
198      }
199    }
200
201    [Storable]
202    private ParameterCollection parameters = new ParameterCollection();
203
204    public IKeyedItemCollection<string, IParameter> Parameters {
205      get { return parameters; }
206    }
207
208    private ReadOnlyKeyedItemCollection<string, IParameter> readOnlyParameters;
209
210    IKeyedItemCollection<string, IParameter> IParameterizedItem.Parameters {
211      get {
212        if (readOnlyParameters == null) readOnlyParameters = parameters.AsReadOnly();
213        return readOnlyParameters;
214      }
215    }
216
217    public IEnumerable<IOptimizer> NestedOptimizers {
218      get { return Enumerable.Empty<IOptimizer>(); }
219    }
220
221    #region Parameter Properties
222
223    public ConstrainedValueParameter<IBenchmark> BenchmarkAlgorithmParameter {
224      get { return (ConstrainedValueParameter<IBenchmark>)Parameters["BenchmarkAlgorithm"]; }
225    }
226
227    private ValueParameter<IntValue> ChunkSizeParameter {
228      get { return (ValueParameter<IntValue>)Parameters["ChunkSize"]; }
229    }
230
231    private ValueParameter<DoubleValue> TimeLimitParameter {
232      get { return (ValueParameter<DoubleValue>)Parameters["TimeLimit"]; }
233    }
234
235    #endregion
236
237    #region Constructors
238
239    public Benchmark() {
240      name = ItemName;
241      description = ItemDescription;
242      parameters = new ParameterCollection();
243      readOnlyParameters = null;
244      executionState = ExecutionState.Stopped;
245      executionTime = TimeSpan.Zero;
246      storeAlgorithmInEachRun = false;
247      runsCounter = 0;
248      Runs = new RunCollection();
249      results = new ResultCollection();
250      CreateParameters();
251      DiscoverBenchmarks();
252      Prepare();
253    }
254
255    public Benchmark(Benchmark original, Cloner cloner) {
256      cloner.RegisterClonedObject(original, this);
257      name = original.name;
258      description = original.description;
259      parameters = cloner.Clone(original.parameters);
260      readOnlyParameters = null;
261      if (ExecutionState == ExecutionState.Started) throw new InvalidOperationException(string.Format("Clone not allowed in execution state \"{0}\".", ExecutionState));
262      executionState = original.executionState;
263      executionTime = original.executionTime;
264      storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
265      runsCounter = original.runsCounter;
266      runs = cloner.Clone(original.runs);
267      Initialize();
268
269      results = cloner.Clone(original.results);
270      DiscoverBenchmarks();
271      Prepare();
272    }
273
274    #endregion
275
276    private void CreateParameters() {
277      Parameters.Add(new ValueParameter<IntValue>("ChunkSize", "The size (MB) of the chunk array that gets generated", new IntValue(0)));
278      Parameters.Add(new ValueParameter<DoubleValue>("TimeLimit", "The time limit (in minutes) for a benchmark run", new DoubleValue(0)));
279    }
280
281    private void DiscoverBenchmarks() {
282      var benchmarks = from t in ApplicationManager.Manager.GetTypes(typeof(IBenchmark))
283                       select t;
284      ItemSet<IBenchmark> values = new ItemSet<IBenchmark>();
285      foreach (var benchmark in benchmarks) {
286        IBenchmark b = (IBenchmark)Activator.CreateInstance(benchmark);
287        values.Add(b);
288      }
289      string paramName = "BenchmarkAlgorithm";
290      if (!Parameters.ContainsKey(paramName)) {
291        if (values.Count > 0) {
292          Parameters.Add(new ConstrainedValueParameter<IBenchmark>(paramName, values, values.First(a => a is IBenchmark)));
293        } else {
294          // throw exception?
295          Parameters.Add(new ConstrainedValueParameter<IBenchmark>(paramName, values));
296        }
297      }
298    }
299
300    private void Initialize() {
301      if (problem != null) RegisterProblemEvents();
302      if (runs != null) RegisterRunsEvents();
303    }
304
305    public virtual void Prepare() {
306      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused) && (ExecutionState != ExecutionState.Stopped))
307        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
308      results.Clear();
309      OnPrepared();
310    }
311
312    public void Prepare(bool clearRuns) {
313      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused) && (ExecutionState != ExecutionState.Stopped))
314        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
315      if (clearRuns) runs.Clear();
316      Prepare();
317    }
318
319    public virtual void Pause() {
320      if (ExecutionState != ExecutionState.Started)
321        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
322    }
323    public virtual void Stop() {
324      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
325        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".", ExecutionState));
326      cancellationTokenSource.Cancel();
327    }
328
329    public virtual void Start() {
330      cancellationTokenSource = new CancellationTokenSource();
331      OnStarted();
332      Task task = Task.Factory.StartNew(Run, cancellationTokenSource.Token, cancellationTokenSource.Token);
333      task.ContinueWith(t => {
334        try {
335          t.Wait();
336        }
337        catch (AggregateException ex) {
338          try {
339            ex.Flatten().Handle(x => x is OperationCanceledException);
340          }
341          catch (AggregateException remaining) {
342            if (remaining.InnerExceptions.Count == 1) OnExceptionOccurred(remaining.InnerExceptions[0]);
343            else OnExceptionOccurred(remaining);
344          }
345        }
346
347        cancellationTokenSource.Dispose();
348        cancellationTokenSource = null;
349        OnStopped();
350      });
351    }
352
353    protected virtual void Run(object state) {
354      CancellationToken cancellationToken = (CancellationToken)state;
355      lastUpdateTime = DateTime.Now;
356      System.Timers.Timer timer = new System.Timers.Timer(250);
357      timer.AutoReset = true;
358      timer.Elapsed += new System.Timers.ElapsedEventHandler(timer_Elapsed);
359      timer.Start();
360      try {
361        BenchmarkAlgorithm = (IBenchmark)BenchmarkAlgorithmParameter.ActualValue;
362        int chunkSize = ((IntValue)ChunkSizeParameter.ActualValue).Value;
363        if (chunkSize > 0) {
364          BenchmarkAlgorithm.ChunkData = CreateDataChuck(chunkSize);
365        }
366        BenchmarkAlgorithm.TimeLimit = TimeSpan.FromMinutes(((DoubleValue)TimeLimitParameter.ActualValue).Value);
367        BenchmarkAlgorithm.Run(cancellationToken, results);
368      }
369      catch (OperationCanceledException) {
370        //Console.WriteLine(ex.ToString());
371      }
372      finally {
373        timer.Elapsed -= new System.Timers.ElapsedEventHandler(timer_Elapsed);
374        timer.Stop();
375        ExecutionTime += DateTime.Now - lastUpdateTime;
376      }
377    }
378
379    private void timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) {
380      System.Timers.Timer timer = (System.Timers.Timer)sender;
381      timer.Enabled = false;
382      DateTime now = DateTime.Now;
383      ExecutionTime += now - lastUpdateTime;
384      lastUpdateTime = now;
385      timer.Enabled = true;
386    }
387
388    public virtual void CollectResultValues(IDictionary<string, IItem> values) {
389      values.Add("Execution Time", new TimeSpanValue(ExecutionTime));
390      CollectResultsRecursively("", Results, values);
391    }
392
393    private void CollectResultsRecursively(string path, ResultCollection results, IDictionary<string, IItem> values) {
394      foreach (IResult result in results) {
395        values.Add(path + result.Name, result.Value);
396        ResultCollection childCollection = result.Value as ResultCollection;
397        if (childCollection != null) {
398          CollectResultsRecursively(path + result.Name + ".", childCollection, values);
399        }
400      }
401    }
402
403    public virtual void CollectParameterValues(IDictionary<string, IItem> values) {
404      foreach (IValueParameter param in parameters.OfType<IValueParameter>()) {
405        if (param.GetsCollected && param.Value != null) values.Add(param.Name, param.Value);
406        if (param.Value is IParameterizedItem) {
407          Dictionary<string, IItem> children = new Dictionary<string, IItem>();
408          ((IParameterizedItem)param.Value).CollectParameterValues(children);
409          foreach (string key in children.Keys)
410            values.Add(param.Name + "." + key, children[key]);
411        }
412      }
413      values.Add("Benchmark Name", new StringValue(Name));
414      values.Add("Benchmark Type", new StringValue(this.GetType().GetPrettyName()));
415    }
416
417    private byte[][] CreateDataChuck(int megaBytes) {
418      if (megaBytes <= 0) {
419        throw new ArgumentException("MegaBytes must be greater than zero", "megaBytes");
420      }
421      byte[][] chunk = new byte[megaBytes][];
422      for (int i = 0; i < chunk.Length; i++) {
423        chunk[i] = new byte[1024 * 1024];
424        random.NextBytes(chunk[i]);
425      }
426      return chunk;
427    }
428    /*
429    private int[] CreateDataChuck(int megaBytes) {
430      if (megaBytes <= 0) {
431        throw new ArgumentException("MegaBytes must be greater than zero", "megaBytes");
432      }
433      int size = (megaBytes * 1020 * 1024) / sizeof(int);
434      int[] data = new int[size];
435      for (int i = 0; i < data.Length; i++) {
436        data[i] = random.Next();
437      }
438      return data;
439    }
440    */
441
442    #region Events
443
444    public event EventHandler ExecutionStateChanged;
445    protected virtual void OnExecutionStateChanged() {
446      EventHandler handler = ExecutionStateChanged;
447      if (handler != null) handler(this, EventArgs.Empty);
448    }
449    public event EventHandler ExecutionTimeChanged;
450    protected virtual void OnExecutionTimeChanged() {
451      EventHandler handler = ExecutionTimeChanged;
452      if (handler != null) handler(this, EventArgs.Empty);
453    }
454    public event EventHandler ProblemChanged;
455    protected virtual void OnProblemChanged() {
456      EventHandler handler = ProblemChanged;
457      if (handler != null) handler(this, EventArgs.Empty);
458    }
459    public event EventHandler StoreAlgorithmInEachRunChanged;
460    protected virtual void OnStoreAlgorithmInEachRunChanged() {
461      EventHandler handler = StoreAlgorithmInEachRunChanged;
462      if (handler != null) handler(this, EventArgs.Empty);
463    }
464    public event EventHandler Prepared;
465    protected virtual void OnPrepared() {
466      ExecutionState = ExecutionState.Prepared;
467      ExecutionTime = TimeSpan.Zero;
468      foreach (IStatefulItem statefulObject in this.GetObjectGraphObjects().OfType<IStatefulItem>()) {
469        statefulObject.InitializeState();
470      }
471      EventHandler handler = Prepared;
472      if (handler != null) handler(this, EventArgs.Empty);
473    }
474    public event EventHandler Started;
475    protected virtual void OnStarted() {
476      ExecutionState = ExecutionState.Started;
477      EventHandler handler = Started;
478      if (handler != null) handler(this, EventArgs.Empty);
479    }
480    public event EventHandler Paused;
481    protected virtual void OnPaused() {
482      ExecutionState = ExecutionState.Paused;
483      EventHandler handler = Paused;
484      if (handler != null) handler(this, EventArgs.Empty);
485    }
486    public event EventHandler Stopped;
487    protected virtual void OnStopped() {
488      ExecutionState = ExecutionState.Stopped;
489      foreach (IStatefulItem statefulObject in this.GetObjectGraphObjects().OfType<IStatefulItem>()) {
490        statefulObject.ClearState();
491      }
492      runsCounter++;
493      runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
494      EventHandler handler = Stopped;
495      if (handler != null) handler(this, EventArgs.Empty);
496    }
497    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
498    protected virtual void OnExceptionOccurred(Exception exception) {
499      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
500      if (handler != null) handler(this, new EventArgs<Exception>(exception));
501    }
502
503    public event EventHandler<CancelEventArgs<string>> NameChanging;
504    protected virtual void OnNameChanging(CancelEventArgs<string> e) {
505      var handler = NameChanging;
506      if (handler != null) handler(this, e);
507    }
508
509    public event EventHandler NameChanged;
510    protected virtual void OnNameChanged() {
511      var handler = NameChanged;
512      if (handler != null) handler(this, EventArgs.Empty);
513      OnToStringChanged();
514    }
515
516    public event EventHandler DescriptionChanged;
517    protected virtual void OnDescriptionChanged() {
518      var handler = DescriptionChanged;
519      if (handler != null) handler(this, EventArgs.Empty);
520    }
521
522    public event EventHandler ItemImageChanged;
523    protected virtual void OnItemImageChanged() {
524      EventHandler handler = ItemImageChanged;
525      if (handler != null) handler(this, EventArgs.Empty);
526    }
527    public event EventHandler ToStringChanged;
528    protected virtual void OnToStringChanged() {
529      EventHandler handler = ToStringChanged;
530      if (handler != null) handler(this, EventArgs.Empty);
531    }
532
533    protected virtual void DeregisterProblemEvents() {
534      problem.OperatorsChanged -= new EventHandler(Problem_OperatorsChanged);
535      problem.Reset -= new EventHandler(Problem_Reset);
536    }
537    protected virtual void RegisterProblemEvents() {
538      problem.OperatorsChanged += new EventHandler(Problem_OperatorsChanged);
539      problem.Reset += new EventHandler(Problem_Reset);
540    }
541    protected virtual void Problem_OperatorsChanged(object sender, EventArgs e) { }
542    protected virtual void Problem_Reset(object sender, EventArgs e) {
543      Prepare();
544    }
545
546    protected virtual void DeregisterRunsEvents() {
547      runs.CollectionReset -= new CollectionItemsChangedEventHandler<IRun>(Runs_CollectionReset);
548    }
549    protected virtual void RegisterRunsEvents() {
550      runs.CollectionReset += new CollectionItemsChangedEventHandler<IRun>(Runs_CollectionReset);
551    }
552    protected virtual void Runs_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
553      runsCounter = runs.Count;
554    }
555
556    #endregion
557
558    #region Clone
559
560    public IDeepCloneable Clone(Cloner cloner) {
561      return new Benchmark(this, cloner);
562    }
563
564    public object Clone() {
565      return Clone(new Cloner());
566    }
567
568    #endregion
569  }
570}
Note: See TracBrowser for help on using the repository browser.