Changeset 6948 for branches/Benchmarking/sources

Timestamp:

11/04/11 13:13:17 (12 years ago)

Author:

spimming

Message:

#1659:

• restructuring of the benchmarking algorithms
• common interface for benchmarks
• 'benchmark' class discovers benchmarking algorithms
• added 'TimeLimit' and 'ChunkSize' parameters to benchmark

Location:

branches/Benchmarking/sources

Files:

• HeuristicLab.Algorithms.Benchmarks.Views/3.3/BenchmarkView.cs (modified) (1 diff)
• HeuristicLab.Algorithms.Benchmarks/3.3/Benchmark.cs (modified) (11 diffs)
• HeuristicLab.Algorithms.Benchmarks/3.3/Dhrystone.cs (deleted)
• HeuristicLab.Algorithms.Benchmarks/3.3/DhrystoneBenchmark.cs (added)
• HeuristicLab.Algorithms.Benchmarks/3.3/HeuristicLab.Algorithms.Benchmarks-3.3.csproj (modified) (1 diff)
• HeuristicLab.Algorithms.Benchmarks/3.3/IBenchmark.cs (added)
• HeuristicLab.Algorithms.Benchmarks/3.3/Linpack.cs (deleted)
• HeuristicLab.Algorithms.Benchmarks/3.3/LinpackBenchmark.cs (added)
• HeuristicLab.Algorithms.Benchmarks/3.3/Whetstone.cs (deleted)
• HeuristicLab.Algorithms.Benchmarks/3.3/WhetstoneAlgorithm.cs (added)

branches/Benchmarking/sources/HeuristicLab.Algorithms.Benchmarks.Views/3.3/BenchmarkView.cs

@@ -26 +26 @@
 namespace HeuristicLab.Algorithms.Benchmarks.Views {
   [View("Benchmark View")]
-  [Content(typeof(Whetstone), true)]
-  [Content(typeof(Dhrystone), true)]
-  [Content(typeof(Linpack), true)]
+  [Content(typeof(Benchmark), true)]
   public partial class BenchmarkView : AlgorithmView {
     public BenchmarkView() {
       InitializeComponent();
       tabControl.TabPages.Remove(this.problemTabPage);
-      tabControl.TabPages.Remove(this.parametersTabPage);
     }
   }

branches/Benchmarking/sources/HeuristicLab.Algorithms.Benchmarks/3.3/Benchmark.cs

@@ -31 +31 @@
 using HeuristicLab.Data;
 using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.PluginInfrastructure;
 
 namespace HeuristicLab.Algorithms.Benchmarks {
@@ -37 +39 @@
   /// A base class for benchmarks.
   /// </summary>
-  [Item("Benchmark", "A base class for benchmarks.")]
+  [Item("Benchmark", "A wrapper for benchmark algorithms.")]
+  [Creatable("Benchmarks")]
   [StorableClass]
-  public abstract class Benchmark : IAlgorithm {
-
+  public class Benchmark : IAlgorithm {
+    private Random random = new Random();
+
+    [Storable]
     private DateTime lastUpdateTime;
+
+    [Storable]
+    private IBenchmark benchmarkAlgorithm;
+    public IBenchmark BenchmarkAlgorithm {
+      get { return benchmarkAlgorithm; }
+      set {
+        if (value == null) throw new ArgumentNullException();
+        benchmarkAlgorithm = value;
+      }
+    }
+
+    private CancellationTokenSource cancellationTokenSource;
 
     [Storable]
@@ -202 +219 @@
     }
 
+    #region Parameter Properties
+
+    public ConstrainedValueParameter<IBenchmark> BenchmarkAlgorithmParameter {
+      get { return (ConstrainedValueParameter<IBenchmark>)Parameters["BenchmarkAlgorithm"]; }
+    }
+
+    private ValueParameter<IntValue> ChunkSizeParameter {
+      get { return (ValueParameter<IntValue>)Parameters["ChunkSize"]; }
+    }
+
+    private ValueParameter<DoubleValue> TimeLimitParameter {
+      get { return (ValueParameter<DoubleValue>)Parameters["TimeLimit"]; }
+    }
+
+    #endregion
+
     #region Constructors
 
-    protected Benchmark() {
+    public Benchmark() {
       name = ItemName;
       description = ItemDescription;
@@ -215 +248 @@
       Runs = new RunCollection();
       results = new ResultCollection();
+      CreateParameters();
+      DiscoverBenchmarks();
       Prepare();
     }
 
-    protected Benchmark(Benchmark original, Cloner cloner) {
+    public Benchmark(Benchmark original, Cloner cloner) {
       cloner.RegisterClonedObject(original, this);
       name = original.name;
@@ -232 +267 @@
       Initialize();
 
-      results = new ResultCollection();
+      results = cloner.Clone(original.results);
+      DiscoverBenchmarks();
       Prepare();
     }
 
     #endregion
+
+    private void CreateParameters() {
+      Parameters.Add(new ValueParameter<IntValue>("ChunkSize", "The size (MB) of the chunk array that gets generated", new IntValue(0)));
+      Parameters.Add(new ValueParameter<DoubleValue>("TimeLimit", "The time limit (in minutes) for a benchmark run", new DoubleValue(0)));
+    }
+
+    private void DiscoverBenchmarks() {
+      var benchmarks = from t in ApplicationManager.Manager.GetTypes(typeof(IBenchmark))
+                       select t;
+      ItemSet<IBenchmark> values = new ItemSet<IBenchmark>();
+      foreach (var benchmark in benchmarks) {
+        IBenchmark b = (IBenchmark)Activator.CreateInstance(benchmark);
+        values.Add(b);
+      }
+      string paramName = "BenchmarkAlgorithm";
+      if (!Parameters.ContainsKey(paramName)) {
+        if (values.Count > 0) {
+          Parameters.Add(new ConstrainedValueParameter<IBenchmark>(paramName, values, values.First(a => a is IBenchmark)));
+        } else {
+          // throw exception?
+          Parameters.Add(new ConstrainedValueParameter<IBenchmark>(paramName, values));
+        }
+      }
+    }
 
     private void Initialize() {
@@ -264 +324 @@
       if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
         throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".", ExecutionState));
+      cancellationTokenSource.Cancel();
     }
 
     public virtual void Start() {
-      var cancellationTokenSource = new CancellationTokenSource();
+      cancellationTokenSource = new CancellationTokenSource();
       OnStarted();
       Task task = Task.Factory.StartNew(Run, cancellationTokenSource.Token, cancellationTokenSource.Token);
@@ -283 +344 @@
           }
         }
+
         cancellationTokenSource.Dispose();
         cancellationTokenSource = null;
@@ -297 +359 @@
       timer.Start();
       try {
-        RunBenchmark();
+        BenchmarkAlgorithm = (IBenchmark)BenchmarkAlgorithmParameter.ActualValue;
+        int chunkSize = ((IntValue)ChunkSizeParameter.ActualValue).Value;
+        if (chunkSize > 0) {
+          BenchmarkAlgorithm.ChunkData = CreateDataChuck(chunkSize);
+        }
+        BenchmarkAlgorithm.TimeLimit = TimeSpan.FromMinutes(((DoubleValue)TimeLimitParameter.ActualValue).Value);
+        BenchmarkAlgorithm.Run(cancellationToken, results);
+      }
+      catch (OperationCanceledException) {
+        //Console.WriteLine(ex.ToString());
       }
       finally {
@@ -304 +375 @@
         ExecutionTime += DateTime.Now - lastUpdateTime;
       }
-
-      cancellationToken.ThrowIfCancellationRequested();
-    }
-
-    protected abstract void RunBenchmark();
+    }
 
     private void timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) {
@@ -348 +415 @@
     }
 
+    private byte[][] CreateDataChuck(int megaBytes) {
+      if (megaBytes <= 0) {
+        throw new ArgumentException("MegaBytes must be greater than zero", "megaBytes");
+      }
+      byte[][] chunk = new byte[megaBytes][];
+      for (int i = 0; i < chunk.Length; i++) {
+        chunk[i] = new byte[1024 * 1024];
+        random.NextBytes(chunk[i]);
+      }
+      return chunk;
+    }
+    /*
+    private int[] CreateDataChuck(int megaBytes) {
+      if (megaBytes <= 0) {
+        throw new ArgumentException("MegaBytes must be greater than zero", "megaBytes");
+      }
+      int size = (megaBytes * 1020 * 1024) / sizeof(int);
+      int[] data = new int[size];
+      for (int i = 0; i < data.Length; i++) {
+        data[i] = random.Next();
+      }
+      return data;
+    }
+    */
+
     #region Events
 
@@ -464 +556 @@
     #endregion
 
-    #region IDeepCloneable Members
-
-    public abstract IDeepCloneable Clone(Cloner cloner);
-
-    #endregion
-
-    #region ICloneable Members
+    #region Clone
+
+    public IDeepCloneable Clone(Cloner cloner) {
+      return new Benchmark(this, cloner);
+    }
 
     public object Clone() {

branches/Benchmarking/sources/HeuristicLab.Algorithms.Benchmarks/3.3/HeuristicLab.Algorithms.Benchmarks-3.3.csproj

@@ -51 +51 @@
       <SubType>Code</SubType>
     </Compile>
-    <Compile Include="Whetstone.cs" />
-    <Compile Include="Dhrystone.cs" />
-    <Compile Include="Linpack.cs" />
+    <Compile Include="DhrystoneBenchmark.cs" />
+    <Compile Include="IBenchmark.cs" />
+    <Compile Include="LinpackBenchmark.cs" />
     <Compile Include="Plugin.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="WhetstoneAlgorithm.cs" />
   </ItemGroup>
   <ItemGroup>