Free cookie consent management tool by TermsFeed Policy Generator

source: branches/2990_VariableImpactBasedFeatureSelection/HeuristicLab.Tests/HeuristicLab.Persistence.Attic/UseCases.cs @ 16724

Last change on this file since 16724 was 16594, checked in by gkronber, 6 years ago

#2520 added unit tests for profiling old vs new persistence with samples
File size: 16.8 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Diagnostics;
25using System.Drawing;
26using System.IO;
27using System.IO.Compression;
28using System.Linq;
29using System.Reflection;
30using System.Threading.Tasks;
31using HEAL.Attic;
32using HeuristicLab.Algorithms.GeneticAlgorithm;
33using HeuristicLab.Analysis;
34using HeuristicLab.Common;
35using HeuristicLab.Core;
36using HeuristicLab.Data;
37using HeuristicLab.Persistence.Core;
38using HeuristicLab.Persistence.Default.Xml;
39using HeuristicLab.Persistence.Interfaces;
40using HeuristicLab.Tests;
41using Microsoft.VisualStudio.TestTools.UnitTesting;
42
43namespace HeuristicLab.Persistence.Attic.Tests {
44  [TestClass]
45  public class UseCases {
46
47    private string tempFile;
48
49    [TestInitialize()]
50    public void CreateTempFile() {
51      tempFile = Path.GetTempFileName();
52    }
53
54    [TestCleanup()]
55    public void ClearTempFile() {
56      StreamReader reader = new StreamReader(tempFile);
57      string s = reader.ReadToEnd();
58      reader.Close();
59      File.Delete(tempFile);
60    }
61
62    [TestMethod]
63    [TestCategory("Persistence.Attic")]
64    [TestProperty("Time", "short")]
65    public void BitmapTest() {
66      Icon icon = System.Drawing.SystemIcons.Hand;
67      Bitmap bitmap = icon.ToBitmap();
68      new ProtoBufSerializer().Serialize(bitmap, tempFile);
69      Bitmap newBitmap = (Bitmap)new ProtoBufSerializer().Deserialize(tempFile);
70
71      Assert.AreEqual(bitmap.Size, newBitmap.Size);
72      for (int i = 0; i < bitmap.Size.Width; i++)
73        for (int j = 0; j < bitmap.Size.Height; j++)
74          Assert.AreEqual(bitmap.GetPixel(i, j), newBitmap.GetPixel(i, j));
75    }
76
77
78    [TestMethod]
79    [TestCategory("Persistence.Attic")]
80    [TestProperty("Time", "short")]
81    public void FontTest() {
82      List<Font> fonts = new List<Font>() {
83        new Font(FontFamily.GenericSansSerif, 12),
84        new Font("Times New Roman", 21, FontStyle.Bold, GraphicsUnit.Pixel),
85        new Font("Courier New", 10, FontStyle.Underline, GraphicsUnit.Document),
86        new Font("Helvetica", 21, FontStyle.Strikeout, GraphicsUnit.Inch, 0, true),
87      };
88      new ProtoBufSerializer().Serialize(fonts, tempFile);
89      var newFonts = (List<Font>)new ProtoBufSerializer().Deserialize(tempFile);
90      Assert.AreEqual(fonts[0], newFonts[0]);
91      Assert.AreEqual(fonts[1], newFonts[1]);
92      Assert.AreEqual(fonts[2], newFonts[2]);
93      Assert.AreEqual(fonts[3], newFonts[3]);
94    }
95
96    [TestMethod]
97    [TestCategory("Persistence.Attic")]
98    [TestProperty("Time", "medium")]
99    public void ConcurrencyTest() {
100      int n = 20;
101      Task[] tasks = new Task[n];
102      for (int i = 0; i < n; i++) {
103        tasks[i] = Task.Factory.StartNew((idx) => {
104          byte[] data;
105          using (var stream = new MemoryStream()) {
106            new ProtoBufSerializer().Serialize(new GeneticAlgorithm(), stream);
107            data = stream.ToArray();
108          }
109        }, i);
110      }
111      Task.WaitAll(tasks);
112    }
113
114    [TestMethod]
115    [TestCategory("Persistence.Attic")]
116    [TestProperty("Time", "medium")]
117    public void ConcurrentBitmapTest() {
118      Bitmap b = new Bitmap(300, 300);
119      System.Random r = new System.Random();
120      for (int x = 0; x < b.Height; x++) {
121        for (int y = 0; y < b.Width; y++) {
122          b.SetPixel(x, y, Color.FromArgb(r.Next()));
123        }
124      }
125      Task[] tasks = new Task[20];
126      byte[][] datas = new byte[tasks.Length][];
127      for (int i = 0; i < tasks.Length; i++) {
128        tasks[i] = Task.Factory.StartNew((idx) => {
129          using (var stream = new MemoryStream()) {
130            new ProtoBufSerializer().Serialize(b, stream);
131            datas[(int)idx] = stream.ToArray();
132          }
133        }, i);
134      }
135      Task.WaitAll(tasks);
136    }
137
138    private void CreateAllSamples() {
139      var asm = this.GetType().Assembly;
140      foreach (var t in asm.GetTypes()) {
141        var attrs = t.GetCustomAttributes<TestClassAttribute>();
142        if (attrs.Any()) {
143          try {
144            var testObj = Activator.CreateInstance(t);
145            foreach (var mi in t.GetMethods(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)) {
146              var mAttrs = mi.GetCustomAttributes<TestCategoryAttribute>();
147              var testCategories = mAttrs.SelectMany(mattr => mattr.TestCategories);
148              if (testCategories.Any(tc => tc == "Samples.Create")) {
149                mi.Invoke(testObj, new object[0]);
150              }
151            }
152          } catch (Exception) { }
153        }
154      }
155    }
156
157    [TestMethod]
158    [TestCategory("Persistence.Attic")]
159    [TestProperty("Time", "long")]
160    public void ProfileSamples() {
161      // CreateAllSamples();
162      var path = SamplesUtils.SamplesDirectory;
163      foreach (var fileName in Directory.EnumerateFiles(path, "*.hl")) {
164        //ProfilePersistenceToMemory(fileName);
165        ProfilePersistenceToDisk(fileName);
166      }
167    }
168
169    private void ProfilePersistenceToMemory(string fileName) {
170      var REPS = 5;
171      var oldDeserStopwatch = new Stopwatch();
172      var oldSerStopwatch = new Stopwatch();
173      var newDeserStopwatch = new Stopwatch();
174      var newSerStopwatch = new Stopwatch();
175      long oldSize = 0, newSize = 0;
176      var config = ConfigurationService.Instance.GetConfiguration(new XmlFormat());
177      int[] oldCollections = new int[3];
178      int[] newCollections = new int[3];
179
180      for (int i = 0; i < REPS; i++) {
181        var original = XmlParser.Deserialize(fileName);
182        object deserializedObject1 = null;
183        object deserializedObject2 = null;
184        byte[] buf;
185        System.GC.Collect();
186        var collection0 = System.GC.CollectionCount(0);
187        var collection1 = System.GC.CollectionCount(1);
188        var collection2 = System.GC.CollectionCount(2);
189        using (var s = new MemoryStream()) {
190
191          oldSerStopwatch.Start();
192          // serialize manually so that the stream won't be closed
193          var serializer = new Core.Serializer(original, config);
194          serializer.InterleaveTypeInformation = true;
195          using (StreamWriter writer = new StreamWriter(new GZipStream(s, CompressionMode.Compress))) {
196            XmlGenerator generator = new XmlGenerator();
197            foreach (ISerializationToken token in serializer) {
198              writer.Write(generator.Format(token));
199            }
200            writer.Flush();
201            oldSize += s.Length;
202          }
203
204          oldSerStopwatch.Stop();
205          buf = s.GetBuffer();
206        }
207        using (var s = new MemoryStream(buf)) {
208          oldDeserStopwatch.Start();
209          deserializedObject1 = XmlParser.Deserialize(s);
210          oldDeserStopwatch.Stop();
211        }
212
213        System.GC.Collect();
214        oldCollections[0] += System.GC.CollectionCount(0) - collection0;
215        oldCollections[1] += System.GC.CollectionCount(1) - collection1;
216        oldCollections[2] += System.GC.CollectionCount(2) - collection2;
217
218        collection0 = System.GC.CollectionCount(0);
219        collection1 = System.GC.CollectionCount(1);
220        collection2 = System.GC.CollectionCount(2);
221
222        // Protobuf only uses Deflate
223        using (var m = new MemoryStream()) {
224          //         using (var s = new GZipStream(m, CompressionMode.Compress)) { // same as old persistence
225          using (var s = new DeflateStream(m, CompressionMode.Compress)) { // new format
226            newSerStopwatch.Start();
227            (new ProtoBufSerializer()).Serialize(original, s, false);
228            s.Flush();
229            newSerStopwatch.Stop();
230            newSize += m.Length;
231          }
232          buf = m.GetBuffer();
233        }
234        using (var m = new MemoryStream(buf)) {
235          // using (var s = new GZipStream(m, CompressionMode.Decompress)) { // same as old persistence
236          using (var s = new DeflateStream(m, CompressionMode.Decompress)) { // new format
237            newDeserStopwatch.Start();
238            deserializedObject2 = (new ProtoBufSerializer()).Deserialize(s);
239            newDeserStopwatch.Stop();
240          }
241        }
242        //Assert.AreEqual(deserializedObject1.GetObjectGraphObjects().Count(), deserializedObject2.GetObjectGraphObjects().Count());
243
244        System.GC.Collect();
245        newCollections[0] += System.GC.CollectionCount(0) - collection0;
246        newCollections[1] += System.GC.CollectionCount(1) - collection1;
247        newCollections[2] += System.GC.CollectionCount(2) - collection2;
248      }
249
250      Console.WriteLine($"{fileName} " +
251        $"{oldSize / (double)REPS} " +
252        $"{newSize / (double)REPS} " +
253        $"{oldSerStopwatch.ElapsedMilliseconds / (double)REPS} " +
254        $"{newSerStopwatch.ElapsedMilliseconds / (double)REPS} " +
255        $"{oldDeserStopwatch.ElapsedMilliseconds / (double)REPS} " +
256        $"{newDeserStopwatch.ElapsedMilliseconds / (double)REPS} " +
257        $"{oldCollections[0] / (double)REPS} " +
258        $"{newCollections[0] / (double)REPS} " +
259        $"{oldCollections[1] / (double)REPS} " +
260        $"{newCollections[1] / (double)REPS} " +
261        $"{oldCollections[2] / (double)REPS} " +
262        $"{newCollections[2] / (double)REPS} " +
263        $"");
264    }
265
266    private void ProfilePersistenceToDisk(string fileName) {
267      var REPS = 5;
268      var oldDeserStopwatch = new Stopwatch();
269      var oldSerStopwatch = new Stopwatch();
270      var newDeserStopwatch = new Stopwatch();
271      var newSerStopwatch = new Stopwatch();
272      long oldSize = 0, newSize = 0;
273      int[] oldCollections = new int[3];
274      int[] newCollections = new int[3];
275
276      for (int i = 0; i < REPS; i++) {
277        var original = XmlParser.Deserialize(fileName);
278        byte[] buf;
279        System.GC.Collect();
280        var collection0 = System.GC.CollectionCount(0);
281        var collection1 = System.GC.CollectionCount(1);
282        var collection2 = System.GC.CollectionCount(2);
283
284        oldSerStopwatch.Start();
285        XmlGenerator.Serialize(original, tempFile);
286        oldSerStopwatch.Stop();
287
288        oldSize += new FileInfo(tempFile).Length;
289        oldDeserStopwatch.Start();
290        var clone = XmlParser.Deserialize(tempFile);
291        oldDeserStopwatch.Stop();
292        System.GC.Collect();
293        oldCollections[0] += System.GC.CollectionCount(0) - collection0;
294        oldCollections[1] += System.GC.CollectionCount(1) - collection1;
295        oldCollections[2] += System.GC.CollectionCount(2) - collection2;
296
297        collection0 = System.GC.CollectionCount(0);
298        collection1 = System.GC.CollectionCount(1);
299        collection2 = System.GC.CollectionCount(2);
300
301        newSerStopwatch.Start();
302        (new ProtoBufSerializer()).Serialize(original, tempFile);
303        newSerStopwatch.Stop();
304        newSize += new FileInfo(tempFile).Length;
305        newDeserStopwatch.Start();
306        var newClone = (new ProtoBufSerializer()).Deserialize(tempFile);
307        newDeserStopwatch.Stop();
308        System.GC.Collect();
309        newCollections[0] += System.GC.CollectionCount(0) - collection0;
310        newCollections[1] += System.GC.CollectionCount(1) - collection1;
311        newCollections[2] += System.GC.CollectionCount(2) - collection2;
312      }
313      Console.WriteLine($"{fileName} " +
314        $"{oldSize / (double)REPS} " +
315        $"{newSize / (double)REPS} " +
316        $"{oldSerStopwatch.ElapsedMilliseconds / (double)REPS} " +
317        $"{newSerStopwatch.ElapsedMilliseconds / (double)REPS} " +
318        $"{oldDeserStopwatch.ElapsedMilliseconds / (double)REPS} " +
319        $"{newDeserStopwatch.ElapsedMilliseconds / (double)REPS} " +
320        $"{oldCollections[0] / (double)REPS} " +
321        $"{newCollections[0] / (double)REPS} " +
322        $"{oldCollections[1] / (double)REPS} " +
323        $"{newCollections[1] / (double)REPS} " +
324        $"{oldCollections[2] / (double)REPS} " +
325        $"{newCollections[2] / (double)REPS} " +
326        $"");
327    }
328
329
330    [TestMethod]
331    [TestCategory("Persistence.Attic")]
332    [TestProperty("Time", "long")]
333    public void TestLoadingSamples() {
334      CreateAllSamples();
335      var path = SamplesUtils.SamplesDirectory;
336      var serializer = new ProtoBufSerializer();
337      foreach (var fileName in Directory.EnumerateFiles(path, "*.hl")) {
338        var original = XmlParser.Deserialize(fileName);
339        var ok = true;
340        foreach (var t in original.GetObjectGraphObjects().Select(o => o.GetType())) {
341          if (
342            t.GetConstructors(BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public)
343              .Any(ctor => StorableConstructorAttribute.IsStorableConstructor(ctor))) {
344            try {
345              if (t.IsGenericType) {
346                var g = Mapper.StaticCache.GetGuid(t.GetGenericTypeDefinition());
347              } else {
348                var g = Mapper.StaticCache.GetGuid(t);
349              }
350            } catch (Exception e) {
351              Console.WriteLine($"type {t.FullName} in {fileName} is not registered with a GUID in HEAL.Attic");
352              ok = false;
353            }
354          }
355        }
356        if (ok) {
357          serializer.Serialize(original, fileName + ".proto");
358          var newVersion = serializer.Deserialize(fileName + ".proto");
359          Console.WriteLine("File: " + fileName);
360          File.Delete(fileName + ".proto");
361        }
362      }
363    }
364
365    [TestMethod]
366    [TestCategory("Persistence.Attic")]
367    [TestProperty("Time", "long")]
368    public void TestLoadingRunAndStoreSamples() {
369      CreateAllSamples();
370      var path = SamplesUtils.SamplesDirectory;
371      var serializer = new ProtoBufSerializer();
372      foreach (var fileName in Directory.EnumerateFiles(path, "*.hl")) {
373        var original = XmlParser.Deserialize(fileName);
374
375        var exec = original as IExecutable;
376        if (exec != null) {
377          exec.Paused += (sender, e) => {
378            serializer.Serialize(exec, fileName + "_paused.proto");
379            Console.WriteLine("Serialized paused file: " + fileName);
380            File.Delete(fileName + "_paused.proto");
381          };
382          exec.Stopped += (sender, e) => {
383            serializer.Serialize(exec, fileName + "_stopped.proto");
384            Console.WriteLine("Serialized stopped file: " + fileName);
385            File.Delete(fileName + "_stopped.proto");
386          };
387          var t = exec.StartAsync();
388          System.Threading.Thread.Sleep(20000); // wait 20 secs
389          if (exec.ExecutionState == ExecutionState.Started) { // only if not already stopped
390            exec.Pause();
391          }
392        }
393      }
394    }
395
396
397    [TestMethod]
398    [TestCategory("Persistence.Attic")]
399    [TestProperty("Time", "short")]
400    public void TestIndexedDataTable() {
401      var dt = new IndexedDataTable<int>("test", "test description");
402      var dr = new IndexedDataRow<int>("test row");
403      dr.Values.Add(Tuple.Create(1, 1.0));
404      dr.Values.Add(Tuple.Create(2, 2.0));
405      dr.Values.Add(Tuple.Create(3, 3.0));
406      dt.Rows.Add(dr);
407      var ser = new ProtoBufSerializer();
408      ser.Serialize(dt, tempFile);
409      var dt2 = (IndexedDataTable<int>)ser.Deserialize(tempFile);
410      Assert.AreEqual(dt.Rows["test row"].Values[0], dt2.Rows["test row"].Values[0]);
411      Assert.AreEqual(dt.Rows["test row"].Values[1], dt2.Rows["test row"].Values[1]);
412      Assert.AreEqual(dt.Rows["test row"].Values[2], dt2.Rows["test row"].Values[2]);
413    }
414
415    [TestMethod]
416    [TestCategory("Persistence.Attic")]
417    [TestProperty("Time", "short")]
418    public void TestPoint2d() {
419      var tag = new IntValue(10);
420      var p = new Point2D<double>(1.0, 2.0, tag);
421      var ser = new ProtoBufSerializer();
422      ser.Serialize(p, tempFile);
423      var p2 = (Point2D<double>)ser.Deserialize(tempFile);
424      Assert.AreEqual(p.X, p2.X);
425      Assert.AreEqual(p.Y, p2.Y);
426      var tag2 = (IntValue)p2.Tag;
427      Assert.AreEqual(tag.Value, tag2.Value);
428    }
429
430    [ClassInitialize]
431    public static void Initialize(TestContext testContext) {
432      ConfigurationService.Instance.Reset();
433    }
434  }
435}
Note: See TracBrowser for help on using the repository browser.