Changeset 3946 for trunk/sources

Timestamp:

06/24/10 15:35:56 (14 years ago)

Author:

epitzer

Message:

Revise CompactNumberArray2StringSerializer: split text into several smaller strings to mitigate memory problems for very large arrays (#1138)

Location:

trunk/sources/HeuristicLab.Persistence/3.3

Files:

• Default/CompositeSerializers/CompactNumberArray2StringSerializer.cs (modified) (6 diffs)
• Tests/UseCases.cs (modified) (1 diff)

trunk/sources/HeuristicLab.Persistence/3.3/Default/CompositeSerializers/CompactNumberArray2StringSerializer.cs

@@ -34 +34 @@
 
   [StorableClass]
-  internal sealed class CompactNumberArray2StringSerializer : ICompositeSerializer {
+  public sealed class CompactNumberArray2StringSerializer : ICompositeSerializer {
+
+    public const int SPLIT_THRESHOLD = 1024 * 1024;
 
     public int Priority {
@@ -60 +62 @@
       int[] lengths = new int[a.Rank];
       int[] lowerBounds = new int[a.Rank];
-      StringBuilder sb = new StringBuilder(a.Rank * 3);
+      StringBuilder sb = new StringBuilder(a.Rank * 6);
       sb.Append(a.Rank).Append(';');
-      int capacity = 1;
       for (int i = 0; i < a.Rank; i++) {
         sb.Append(a.GetLength(i)).Append(';');
         lengths[i] = a.GetLength(i);
-        capacity *= lengths[i];
       }
-      sb.EnsureCapacity(capacity * 3);
       for (int i = 0; i < a.Rank; i++) {
         sb.Append(a.GetLowerBound(i)).Append(';');
         lowerBounds[i] = a.GetLowerBound(i);
       }
+      yield return new Tag(sb.ToString());
+      int nElements = 1;
+      for (int i = 0; i < a.Rank; i++) {
+        lowerBounds[i] = a.GetLowerBound(i);
+        lengths[i] = a.GetLength(i);
+        nElements *= lengths[i];
+      }
+      sb = new StringBuilder(Math.Min(nElements * 3, SPLIT_THRESHOLD));
       int[] positions = (int[])lowerBounds.Clone();
       while (positions[a.Rank - 1] < lengths[a.Rank - 1] + lowerBounds[a.Rank - 1]) {
         sb.Append(numberConverter.Format(a.GetValue(positions))).Append(';');
+        if (sb.Length > SPLIT_THRESHOLD && sb.Length > sb.Capacity - 18) {
+          yield return new Tag(sb.ToString());
+          sb = new StringBuilder(Math.Min(nElements * 3, SPLIT_THRESHOLD));
+        }
         positions[0] += 1;
         for (int i = 0; i < a.Rank - 1; i++) {
@@ -86 +97 @@
         }
       }
-      yield return new Tag("compact array", sb.ToString());
+      if (sb.Length > 0)
+        yield return new Tag(sb.ToString());
     }
 
+    private static Tag[] emptyTag = new Tag[0];
     public IEnumerable<Tag> Decompose(object obj) {
-      return new Tag[] { };
+      return emptyTag;
     }
 
@@ -99 +112 @@
         var valueIter = ((string)tagIter.Current.Value).GetSplitEnumerator(';');
         valueIter.MoveNext();
-        int rank = int.Parse((string)valueIter.Current);
+        int rank = int.Parse(valueIter.Current);
         int[] lengths = new int[rank];
         int[] lowerBounds = new int[rank];
         for (int i = 0; i < rank; i++) {
           valueIter.MoveNext();
-          lengths[i] = int.Parse((string)valueIter.Current);
+          lengths[i] = int.Parse(valueIter.Current);
         }
         for (int i = 0; i < rank; i++) {
           valueIter.MoveNext();
-          lowerBounds[i] = int.Parse((string)valueIter.Current);
+          lowerBounds[i] = int.Parse(valueIter.Current);
         }
         Type elementType = type.GetElementType();
         Array a = Array.CreateInstance(elementType, lengths, lowerBounds);
+        if (a == null) throw new PersistenceException("invalid instance data type, expected array");
         int[] positions = (int[])lowerBounds.Clone();
-        while (valueIter.MoveNext()) {
-          a.SetValue(
-            numberConverter.Parse((string)valueIter.Current, elementType),
-            positions);
-          positions[0] += 1;
-          for (int i = 0; i < rank - 1; i++) {
-            if (positions[i] >= lengths[i] + lowerBounds[i]) {
-              positions[i + 1] += 1;
-              positions[i] = lowerBounds[i];
-            } else {
-              break;
+        while (tagIter.MoveNext()) {
+          valueIter = ((string)tagIter.Current.Value).GetSplitEnumerator(';');
+          while (valueIter.MoveNext()) {
+            a.SetValue(numberConverter.Parse(valueIter.Current, elementType), positions);
+            positions[0] += 1;
+            for (int i = 0; i < a.Rank - 1; i++) {
+              if (positions[i] >= lengths[i] + lowerBounds[i]) {
+                positions[i + 1] += 1;
+                positions[i] = lowerBounds[i];
+              } else {
+                break;
+              }
             }
           }
@@ -129 +144 @@
         return a;
       } catch (InvalidOperationException e) {
-        throw new PersistenceException("Insufficient data to deserialize compact array", e);
+        throw new PersistenceException("Insuffictient data to deserialize compact array", e);
       } catch (InvalidCastException e) {
         throw new PersistenceException("Invalid element data during compact array deserialization", e);
@@ -136 +151 @@
 
     public void Populate(object instance, IEnumerable<Tag> tags, Type type) {
-      // Nothing to do: Compact arrays are already populated during instance creation.
+      // Nothing to do. Arrays are populated during instance creation;
     }
 

trunk/sources/HeuristicLab.Persistence/3.3/Tests/UseCases.cs

@@ -1075 +1075 @@
     }
 
+    [TestMethod]
+    public void TestCompactNumberArraySerializer() {
+      Random r = new Random();
+      double[] a = new double[CompactNumberArray2StringSerializer.SPLIT_THRESHOLD * 2 + 1];
+      for (int i = 0; i < a.Length; i++)
+        a[i] = r.Next(10);
+      var config = ConfigurationService.Instance.GetDefaultConfig(new XmlFormat());
+      config = new Configuration(config.Format,
+        config.PrimitiveSerializers.Where(s => s.SourceType != typeof(double[])),
+        config.CompositeSerializers);
+      XmlGenerator.Serialize(a, tempFile, config);
+      double[] newA = XmlParser.Deserialize<double[]>(tempFile);
+      Assert.AreEqual(a.Length, newA.Length);
+      for (int i = 0; i < a.Rank; i++) {
+        Assert.AreEqual(a.GetLength(i), newA.GetLength(i));
+        Assert.AreEqual(a.GetLowerBound(i), newA.GetLowerBound(i));
+      }
+      for (int i = 0; i < a.Length; i++) {
+        Assert.AreEqual(a[i], newA[i]);
+      }
+    }
 
     [ClassInitialize]