Free cookie consent management tool by TermsFeed Policy Generator

source: branches/WebJobManager/HeuristicLab.Algorithms.Benchmarks/3.3/BenchmarkAlgorithm.cs @ 13808

Last change on this file since 13808 was 13656, checked in by ascheibe, 9 years ago

#2582 created branch for Hive Web Job Manager

File size: 19.2 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Drawing;
25using System.Linq;
26using System.Threading;
27using System.Threading.Tasks;
28using HeuristicLab.Collections;
29using HeuristicLab.Common;
30using HeuristicLab.Core;
31using HeuristicLab.Data;
32using HeuristicLab.Optimization;
33using HeuristicLab.Parameters;
34using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
35using HeuristicLab.PluginInfrastructure;
36
37namespace HeuristicLab.Algorithms.Benchmarks {
38  [Item("Benchmark Algorithm", "An algorithm to execute performance benchmarks (Linpack, Dhrystone, Whetstone, etc.).")]
39  [Creatable(CreatableAttribute.Categories.TestingAndAnalysis, Priority = 130)]
40  [StorableClass]
41  public sealed class BenchmarkAlgorithm : IAlgorithm {
42    private CancellationTokenSource cancellationTokenSource;
43
44    public string ItemName
45    {
46      get { return ItemAttribute.GetName(this.GetType()); }
47    }
48    public string ItemDescription
49    {
50      get { return ItemAttribute.GetDescription(this.GetType()); }
51    }
52    public Version ItemVersion
53    {
54      get { return ItemAttribute.GetVersion(this.GetType()); }
55    }
56    public static Image StaticItemImage
57    {
58      get { return new Bitmap(25, 25); }
59    }
60    public Image ItemImage
61    {
62      get
63      {
64        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
65        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
66        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
67        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
68        else return ItemAttribute.GetImage(this.GetType());
69      }
70    }
71
72    [Storable]
73    private DateTime lastUpdateTime;
74
75    [Storable]
76    private IBenchmark benchmark;
77    public IBenchmark Benchmark
78    {
79      get { return benchmark; }
80      set
81      {
82        if (value == null) throw new ArgumentNullException();
83        benchmark = value;
84      }
85    }
86
87    [Storable]
88    private ExecutionState executionState;
89    public ExecutionState ExecutionState
90    {
91      get { return executionState; }
92      private set
93      {
94        if (executionState != value) {
95          executionState = value;
96          OnExecutionStateChanged();
97          OnItemImageChanged();
98        }
99      }
100    }
101
102    [Storable]
103    private TimeSpan executionTime;
104    public TimeSpan ExecutionTime
105    {
106      get { return executionTime; }
107      private set
108      {
109        executionTime = value;
110        OnExecutionTimeChanged();
111      }
112    }
113
114    [Storable]
115    private bool storeAlgorithmInEachRun;
116    public bool StoreAlgorithmInEachRun
117    {
118      get { return storeAlgorithmInEachRun; }
119      set
120      {
121        if (storeAlgorithmInEachRun != value) {
122          storeAlgorithmInEachRun = value;
123          OnStoreAlgorithmInEachRunChanged();
124        }
125      }
126    }
127
128    [Storable]
129    private int runsCounter;
130
131    [Storable]
132    private RunCollection runs = new RunCollection();
133    public RunCollection Runs
134    {
135      get { return runs; }
136      private set
137      {
138        if (value == null) throw new ArgumentNullException();
139        if (runs != value) {
140          if (runs != null) DeregisterRunsEvents();
141          runs = value;
142          if (runs != null) RegisterRunsEvents();
143        }
144      }
145    }
146
147    [Storable]
148    private ResultCollection results;
149    public ResultCollection Results
150    {
151      get { return results; }
152    }
153
154    public Type ProblemType
155    {
156      get
157      {
158        // BenchmarkAlgorithm does not have a problem, so return a type which is no problem for sure
159        return typeof(BenchmarkAlgorithm);
160      }
161    }
162
163    public IProblem Problem
164    {
165      get { return null; }
166      set { throw new NotImplementedException("BenchmarkAlgorithm does not have a problem."); }
167    }
168
169    [Storable]
170    private string name;
171    public string Name
172    {
173      get { return name; }
174      set
175      {
176        if (!CanChangeName) throw new NotSupportedException("Name cannot be changed.");
177        if (!(name.Equals(value) || (value == null) && (name == string.Empty))) {
178          CancelEventArgs<string> e = value == null ? new CancelEventArgs<string>(string.Empty) : new CancelEventArgs<string>(value);
179          OnNameChanging(e);
180          if (!e.Cancel) {
181            name = value == null ? string.Empty : value;
182            OnNameChanged();
183            runs.OptimizerName = name;
184          }
185        }
186      }
187    }
188    public bool CanChangeName
189    {
190      get { return true; }
191    }
192
193    [Storable]
194    private string description;
195    public string Description
196    {
197      get { return description; }
198      set
199      {
200        if (!CanChangeDescription) throw new NotSupportedException("Description cannot be changed.");
201        if (!(description.Equals(value) || (value == null) && (description == string.Empty))) {
202          description = value == null ? string.Empty : value;
203          OnDescriptionChanged();
204        }
205      }
206    }
207    public bool CanChangeDescription
208    {
209      get { return true; }
210    }
211
212    [Storable]
213    private ParameterCollection parameters = new ParameterCollection();
214    public IKeyedItemCollection<string, IParameter> Parameters
215    {
216      get { return parameters; }
217    }
218    private ReadOnlyKeyedItemCollection<string, IParameter> readOnlyParameters;
219    IKeyedItemCollection<string, IParameter> IParameterizedItem.Parameters
220    {
221      get
222      {
223        if (readOnlyParameters == null) readOnlyParameters = parameters.AsReadOnly();
224        return readOnlyParameters;
225      }
226    }
227
228    public IEnumerable<IOptimizer> NestedOptimizers
229    {
230      get { return Enumerable.Empty<IOptimizer>(); }
231    }
232
233    #region Parameter Properties
234    public IConstrainedValueParameter<IBenchmark> BenchmarkParameter
235    {
236      get { return (IConstrainedValueParameter<IBenchmark>)Parameters["Benchmark"]; }
237    }
238    private ValueParameter<IntValue> ChunkSizeParameter
239    {
240      get { return (ValueParameter<IntValue>)Parameters["ChunkSize"]; }
241    }
242    private ValueParameter<DoubleValue> TimeLimitParameter
243    {
244      get { return (ValueParameter<DoubleValue>)Parameters["TimeLimit"]; }
245    }
246    #endregion
247
248    #region Constructors
249    [StorableConstructor]
250    private BenchmarkAlgorithm(bool deserializing) { }
251    private BenchmarkAlgorithm(BenchmarkAlgorithm original, Cloner cloner) {
252      if (original.ExecutionState == ExecutionState.Started) throw new InvalidOperationException(string.Format("Clone not allowed in execution state \"{0}\".", ExecutionState));
253      cloner.RegisterClonedObject(original, this);
254      name = original.name;
255      description = original.description;
256      parameters = cloner.Clone(original.parameters);
257      readOnlyParameters = null;
258      executionState = original.executionState;
259      executionTime = original.executionTime;
260      storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
261      runsCounter = original.runsCounter;
262      Runs = cloner.Clone(original.runs);
263      results = cloner.Clone(original.results);
264    }
265    public BenchmarkAlgorithm() {
266      name = ItemName;
267      description = ItemDescription;
268      parameters = new ParameterCollection();
269      readOnlyParameters = null;
270      executionState = ExecutionState.Stopped;
271      executionTime = TimeSpan.Zero;
272      storeAlgorithmInEachRun = false;
273      runsCounter = 0;
274      Runs = new RunCollection() { OptimizerName = name };
275      results = new ResultCollection();
276      CreateParameters();
277      DiscoverBenchmarks();
278      Prepare();
279    }
280    #endregion
281
282    private void CreateParameters() {
283      Parameters.Add(new ValueParameter<IntValue>("ChunkSize", "The size in MB of the chunk data array that is generated.", new IntValue(0)));
284      Parameters.Add(new ValueParameter<DoubleValue>("TimeLimit", "The time limit in minutes for a benchmark run (zero means a fixed number of iterations).", new DoubleValue(0)));
285    }
286    private void DiscoverBenchmarks() {
287      var benchmarks = from t in ApplicationManager.Manager.GetTypes(typeof(IBenchmark))
288                       select t;
289      ItemSet<IBenchmark> values = new ItemSet<IBenchmark>();
290      foreach (var benchmark in benchmarks) {
291        IBenchmark b = (IBenchmark)Activator.CreateInstance(benchmark);
292        values.Add(b);
293      }
294      string paramName = "Benchmark";
295      if (!Parameters.ContainsKey(paramName)) {
296        if (values.Count > 0) {
297          Parameters.Add(new ConstrainedValueParameter<IBenchmark>(paramName, "The benchmark which should be executed.", values, values.First(a => a is IBenchmark)));
298        } else {
299          Parameters.Add(new ConstrainedValueParameter<IBenchmark>(paramName, "The benchmark which should be executed.", values));
300        }
301      }
302    }
303
304    public IDeepCloneable Clone(Cloner cloner) {
305      return new BenchmarkAlgorithm(this, cloner);
306    }
307    public object Clone() {
308      return Clone(new Cloner());
309    }
310
311    public override string ToString() {
312      return Name;
313    }
314
315    public void Prepare() {
316      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused) && (ExecutionState != ExecutionState.Stopped))
317        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
318      results.Clear();
319      OnPrepared();
320    }
321    public void Prepare(bool clearRuns) {
322      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused) && (ExecutionState != ExecutionState.Stopped))
323        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
324      if (clearRuns) runs.Clear();
325      Prepare();
326    }
327    public void Pause() {
328      if (ExecutionState != ExecutionState.Started)
329        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
330    }
331    public void Stop() {
332      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
333        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".", ExecutionState));
334      cancellationTokenSource.Cancel();
335    }
336    public void Start() {
337      cancellationTokenSource = new CancellationTokenSource();
338      OnStarted();
339      Task task = Task.Factory.StartNew(Run, cancellationTokenSource.Token, cancellationTokenSource.Token);
340      task.ContinueWith(t => {
341        try {
342          t.Wait();
343        }
344        catch (AggregateException ex) {
345          try {
346            ex.Flatten().Handle(x => x is OperationCanceledException);
347          }
348          catch (AggregateException remaining) {
349            if (remaining.InnerExceptions.Count == 1) OnExceptionOccurred(remaining.InnerExceptions[0]);
350            else OnExceptionOccurred(remaining);
351          }
352        }
353
354        cancellationTokenSource.Dispose();
355        cancellationTokenSource = null;
356        OnStopped();
357      });
358    }
359
360    private void Run(object state) {
361      CancellationToken cancellationToken = (CancellationToken)state;
362      lastUpdateTime = DateTime.UtcNow;
363      System.Timers.Timer timer = new System.Timers.Timer(250);
364      timer.AutoReset = true;
365      timer.Elapsed += new System.Timers.ElapsedEventHandler(timer_Elapsed);
366      timer.Start();
367      try {
368        Benchmark = (IBenchmark)BenchmarkParameter.ActualValue;
369        int chunkSize = ((IntValue)ChunkSizeParameter.ActualValue).Value;
370        if (chunkSize > 0) {
371          Benchmark.ChunkData = CreateChunkData(chunkSize);
372        } else if (chunkSize < 0) {
373          throw new ArgumentException("ChunkSize must not be negativ.");
374        }
375        TimeSpan timelimit = TimeSpan.FromMinutes(((DoubleValue)TimeLimitParameter.ActualValue).Value);
376        if (timelimit.TotalMilliseconds < 0) {
377          throw new ArgumentException("TimeLimit must not be negativ. ");
378        }
379        Benchmark.TimeLimit = timelimit;
380        Benchmark.Run(cancellationToken, results);
381      }
382      catch (OperationCanceledException) {
383      }
384      finally {
385        timer.Elapsed -= new System.Timers.ElapsedEventHandler(timer_Elapsed);
386        timer.Stop();
387        ExecutionTime += DateTime.UtcNow - lastUpdateTime;
388      }
389    }
390
391    private void timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) {
392      System.Timers.Timer timer = (System.Timers.Timer)sender;
393      timer.Enabled = false;
394      DateTime now = DateTime.UtcNow;
395      ExecutionTime += now - lastUpdateTime;
396      lastUpdateTime = now;
397      timer.Enabled = true;
398    }
399
400    public void CollectResultValues(IDictionary<string, IItem> values) {
401      values.Add("Execution Time", new TimeSpanValue(ExecutionTime));
402      CollectResultsRecursively("", Results, values);
403    }
404    private void CollectResultsRecursively(string path, ResultCollection results, IDictionary<string, IItem> values) {
405      foreach (IResult result in results) {
406        values.Add(path + result.Name, result.Value);
407        ResultCollection childCollection = result.Value as ResultCollection;
408        if (childCollection != null) {
409          CollectResultsRecursively(path + result.Name + ".", childCollection, values);
410        }
411      }
412    }
413    public void CollectParameterValues(IDictionary<string, IItem> values) {
414      foreach (IValueParameter param in parameters.OfType<IValueParameter>()) {
415        if (param.GetsCollected && param.Value != null) values.Add(param.Name, param.Value);
416        if (param.Value is IParameterizedItem) {
417          Dictionary<string, IItem> children = new Dictionary<string, IItem>();
418          ((IParameterizedItem)param.Value).CollectParameterValues(children);
419          foreach (string key in children.Keys)
420            values.Add(param.Name + "." + key, children[key]);
421        }
422      }
423    }
424
425    private byte[][] CreateChunkData(int megaBytes) {
426      if (megaBytes <= 0) {
427        throw new ArgumentException("MegaBytes must be greater than zero", "megaBytes");
428      }
429      Random random = new Random();
430      byte[][] chunk = new byte[megaBytes][];
431      for (int i = 0; i < chunk.Length; i++) {
432        chunk[i] = new byte[1024 * 1024];
433        random.NextBytes(chunk[i]);
434      }
435      return chunk;
436    }
437
438    #region Events
439    public event EventHandler ExecutionStateChanged;
440    private void OnExecutionStateChanged() {
441      EventHandler handler = ExecutionStateChanged;
442      if (handler != null) handler(this, EventArgs.Empty);
443    }
444    public event EventHandler ExecutionTimeChanged;
445    private void OnExecutionTimeChanged() {
446      EventHandler handler = ExecutionTimeChanged;
447      if (handler != null) handler(this, EventArgs.Empty);
448    }
449    public event EventHandler ProblemChanged { add { } remove { } }
450    public event EventHandler StoreAlgorithmInEachRunChanged;
451    private void OnStoreAlgorithmInEachRunChanged() {
452      EventHandler handler = StoreAlgorithmInEachRunChanged;
453      if (handler != null) handler(this, EventArgs.Empty);
454    }
455    public event EventHandler Prepared;
456    private void OnPrepared() {
457      ExecutionState = ExecutionState.Prepared;
458      ExecutionTime = TimeSpan.Zero;
459      foreach (IStatefulItem statefulObject in this.GetObjectGraphObjects().OfType<IStatefulItem>()) {
460        statefulObject.InitializeState();
461      }
462      EventHandler handler = Prepared;
463      if (handler != null) handler(this, EventArgs.Empty);
464    }
465    public event EventHandler Started;
466    private void OnStarted() {
467      ExecutionState = ExecutionState.Started;
468      EventHandler handler = Started;
469      if (handler != null) handler(this, EventArgs.Empty);
470    }
471    public event EventHandler Paused;
472    private void OnPaused() {
473      ExecutionState = ExecutionState.Paused;
474      EventHandler handler = Paused;
475      if (handler != null) handler(this, EventArgs.Empty);
476    }
477    public event EventHandler Stopped;
478    private void OnStopped() {
479      ExecutionState = ExecutionState.Stopped;
480      foreach (IStatefulItem statefulObject in this.GetObjectGraphObjects().OfType<IStatefulItem>()) {
481        statefulObject.ClearState();
482      }
483      runsCounter++;
484      runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
485      EventHandler handler = Stopped;
486      if (handler != null) handler(this, EventArgs.Empty);
487    }
488    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
489    private void OnExceptionOccurred(Exception exception) {
490      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
491      if (handler != null) handler(this, new EventArgs<Exception>(exception));
492    }
493
494    public event EventHandler<CancelEventArgs<string>> NameChanging;
495    private void OnNameChanging(CancelEventArgs<string> e) {
496      var handler = NameChanging;
497      if (handler != null) handler(this, e);
498    }
499
500    public event EventHandler NameChanged;
501    private void OnNameChanged() {
502      var handler = NameChanged;
503      if (handler != null) handler(this, EventArgs.Empty);
504      OnToStringChanged();
505    }
506
507    public event EventHandler DescriptionChanged;
508    private void OnDescriptionChanged() {
509      var handler = DescriptionChanged;
510      if (handler != null) handler(this, EventArgs.Empty);
511    }
512
513    public event EventHandler ItemImageChanged;
514    private void OnItemImageChanged() {
515      EventHandler handler = ItemImageChanged;
516      if (handler != null) handler(this, EventArgs.Empty);
517    }
518    public event EventHandler ToStringChanged;
519    private void OnToStringChanged() {
520      EventHandler handler = ToStringChanged;
521      if (handler != null) handler(this, EventArgs.Empty);
522    }
523
524    private void DeregisterRunsEvents() {
525      runs.CollectionReset -= new CollectionItemsChangedEventHandler<IRun>(Runs_CollectionReset);
526    }
527    private void RegisterRunsEvents() {
528      runs.CollectionReset += new CollectionItemsChangedEventHandler<IRun>(Runs_CollectionReset);
529    }
530    private void Runs_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
531      runsCounter = runs.Count;
532    }
533    #endregion
534  }
535}
Note: See TracBrowser for help on using the repository browser.