Changeset 3005 for trunk

Timestamp:

03/11/10 15:50:50 (14 years ago)

Author:

epitzer

Message:

add support for type information interleaving and subsequently true streaming (#548)

Location:

trunk/sources/HeuristicLab.Persistence/3.3

Files:

• Core/DeSerializer.cs (modified) (3 diffs)
• Core/GeneratorBase.cs (modified) (2 diffs)
• Core/Serializer.cs (modified) (5 diffs)
• Core/Tokens/TypeToken.cs (added)
• Default/DebugString/DebugStringGenerator.cs (modified) (1 diff)
• Default/Xml/XmlGenerator.cs (modified) (9 diffs)
• Default/Xml/XmlParser.cs (modified) (7 diffs)
• HeuristicLab.Persistence-3.3.csproj (modified) (1 diff)
• Tests/UseCases.cs (modified) (3 diffs)

trunk/sources/HeuristicLab.Persistence/3.3/Core/DeSerializer.cs

@@ -77 +77 @@
       parentStack = new Stack<Midwife>();
       typeIds = new Dictionary<int, Type>();
-      serializerMapping = CreateSerializers(typeCache);
-    }
-
-    private Dictionary<Type, object> CreateSerializers(IEnumerable<TypeMapping> typeCache) {
-      Dictionary<Type, object> serializerInstances = new Dictionary<Type, object>();
+      serializerMapping = new Dictionary<Type, object>();
+      foreach (var typeMapping in typeCache) {
+        AddTypeInfo(typeMapping);
+      }
+    }
+
+    private Dictionary<Type, object> serializerInstances = new Dictionary<Type, object>();    
+
+    public void AddTypeInfo(TypeMapping typeMapping) {
+      if (typeIds.ContainsKey(typeMapping.Id))
+        return;
       try {
-        var map = new Dictionary<Type, object>();
-        foreach (var typeMapping in typeCache) {
-          Type type = TypeLoader.Load(typeMapping.TypeName);
-          typeIds.Add(typeMapping.Id, type);
-          Type serializerType = TypeLoader.Load(typeMapping.Serializer);
-          object serializer;
-          if (serializerInstances.ContainsKey(serializerType))
-            serializer = serializerInstances[serializerType];
-          else
-            serializer = Activator.CreateInstance(serializerType, true);
-          map.Add(type, serializer);
-        }
-        return map;
+        Type type = TypeLoader.Load(typeMapping.TypeName);
+        typeIds.Add(typeMapping.Id, type);
+        Type serializerType = TypeLoader.Load(typeMapping.Serializer);
+        object serializer;
+        if (serializerInstances.ContainsKey(serializerType))
+          serializer = serializerInstances[serializerType];
+        else
+          serializer = Activator.CreateInstance(serializerType, true);
+        serializerMapping.Add(type, serializer);
       } catch (PersistenceException) {
         throw;
       } catch (Exception e) {
-        throw new PersistenceException(
-          "The serialization type cache could not be loaded.\r\n" +
-          "This usualy happens when you are missing an Assembly or Plugin.", e);
+        throw new PersistenceException(string.Format(
+          "Could not add type info for {0} ({1})",
+          typeMapping.TypeName, typeMapping.Serializer), e);
       }
     }
@@ -125 +127 @@
         } else if (t == typeof(MetaInfoEndToken)) {
           MetaInfoEnd((MetaInfoEndToken)token);
+        } else if (t == typeof(TypeToken)) {
+          Type((TypeToken)token);
         } else {
           throw new PersistenceException("invalid token type");
@@ -138 +142 @@
       if (!m.MetaMode && m.Obj == null)
         CreateInstance(m);
+    }
+
+    private void Type(TypeToken token) {
+      AddTypeInfo(new TypeMapping(token.Id, token.TypeName, token.Serializer));
     }
 

trunk/sources/HeuristicLab.Persistence/3.3/Core/GeneratorBase.cs

@@ -34 +34 @@
       if (type == typeof(MetaInfoEndToken))
         return Format((MetaInfoEndToken)token);
+      if (type == typeof(TypeToken))
+        return Format((TypeToken)token);
       throw new ApplicationException("Invalid token of type " + type.FullName);
     }
@@ -44 +46 @@
     protected abstract T Format(MetaInfoBeginToken metaInfoBeginToken);
     protected abstract T Format(MetaInfoEndToken metaInfoEndToken);
+    protected abstract T Format(TypeToken typeToken);
 
   }

trunk/sources/HeuristicLab.Persistence/3.3/Core/Serializer.cs

@@ -49 +49 @@
     private readonly bool isTestRun;
     private readonly List<Exception> exceptions;
+
+    public bool InterleaveTypeInformation { get; set; }
 
     /// <summary>
@@ -106 +108 @@
     /// don't stop at the first exception</param>
     public Serializer(object obj, Configuration configuration, string rootName, bool isTestRun) {
+      this.InterleaveTypeInformation = false;
       this.obj = obj;
       this.rootName = rootName;
@@ -144 +147 @@
       if (obj2id.ContainsKey(value))
         return ReferenceEnumerator(accessor.Name, obj2id[value]);
-      if (!typeCache.ContainsKey(type))
+      bool emitTypeInfo = false;
+      if (!typeCache.ContainsKey(type)) {
         typeCache.Add(type, typeCache.Count);
+        emitTypeInfo = InterleaveTypeInformation;
+      }
       int typeId = typeCache[type];
       int? id = null;
@@ -155 +161 @@
         IPrimitiveSerializer primitiveSerializer = configuration.GetPrimitiveSerializer(type);
         if (primitiveSerializer != null)
-          return PrimitiveEnumerator(accessor.Name, typeId, primitiveSerializer.Format(value), id);
+          return PrimitiveEnumerator(
+            accessor.Name,
+            typeId,
+            primitiveSerializer.Format(value),
+            id,
+            emitTypeInfo);
         ICompositeSerializer compositeSerializer = configuration.GetCompositeSerializer(type);
         if (compositeSerializer != null)
-          return CompositeEnumerator(accessor.Name, compositeSerializer.Decompose(value), id, typeId, compositeSerializer.CreateMetaInfo(value));
+          return CompositeEnumerator(
+            accessor.Name,
+            compositeSerializer.Decompose(value),
+            id,
+            typeId,
+            compositeSerializer.CreateMetaInfo(value),
+            emitTypeInfo);
         throw CreatePersistenceException(type);
       } catch (Exception x) {
@@ -200 +217 @@
 
     private IEnumerator<ISerializationToken> PrimitiveEnumerator(string name,
-        int typeId, ISerialData serializedValue, int? id) {
+        int typeId, ISerialData serializedValue, int? id, bool emitTypeInfo) {
+      if (emitTypeInfo) {
+        var mapping = TypeCache[typeId];
+        yield return new TypeToken(mapping.Id, mapping.TypeName, mapping.Serializer);
+      }
       yield return new PrimitiveToken(name, typeId, id, serializedValue);
     }
 
     private IEnumerator<ISerializationToken> CompositeEnumerator(
-        string name, IEnumerable<Tag> tags, int? id, int typeId, IEnumerable<Tag> metaInfo) {
+        string name, IEnumerable<Tag> tags, int? id, int typeId, IEnumerable<Tag> metaInfo,
+        bool emitTypeInfo) {
+      if (emitTypeInfo) {
+        var mapping = TypeCache[typeId];
+        yield return new TypeToken(mapping.Id, mapping.TypeName, mapping.Serializer);
+      }
       yield return new BeginToken(name, typeId, id);
       bool first = true;

trunk/sources/HeuristicLab.Persistence/3.3/Default/DebugString/DebugStringGenerator.cs

@@ -97 +97 @@
     }
 
+    protected override string Format(TypeToken typeToken) {
+      return string.Empty;
+    }
+
     public static string Serialize(object o) {
       return Serialize(o, ConfigurationService.Instance.GetDefaultConfig(new DebugStringFormat()));

trunk/sources/HeuristicLab.Persistence/3.3/Default/Xml/XmlGenerator.cs

@@ -8 +8 @@
 using HeuristicLab.Tracing;
 using HeuristicLab.Persistence.Core.Tokens;
+using System.IO.Compression;
 
 namespace HeuristicLab.Persistence.Default.Xml {
@@ -109 +110 @@
 
     protected override string Format(BeginToken beginToken) {
-      return CreateNodeStart(
-        XmlStringConstants.COMPOSITE,
-        new Dictionary<string, object> {
+      var dict = new Dictionary<string, object> {
           {"name", beginToken.Name},
           {"typeId", beginToken.TypeId},
-          {"id", beginToken.Id}});
+          {"id", beginToken.Id}};
+      AddTypeInfo(beginToken.TypeId, dict);
+      return CreateNodeStart(XmlStringConstants.COMPOSITE, dict);
+        
+    }
+
+    private void AddTypeInfo(int typeId, Dictionary<string, object> dict) {
+      if (lastTypeToken != null) {
+        if (typeId == lastTypeToken.Id) {
+          dict.Add("typeName", lastTypeToken.TypeName);
+          dict.Add("serializer", lastTypeToken.Serializer);
+          lastTypeToken = null;
+        } else {
+          FlushTypeToken();
+        }
+      }
     }
 
@@ -122 +136 @@
 
     protected override string Format(PrimitiveToken dataToken) {
-      return CreateNode(XmlStringConstants.PRIMITIVE,
-        new Dictionary<string, object> {
+      var dict = new Dictionary<string, object> {
             {"typeId", dataToken.TypeId},
             {"name", dataToken.Name},
-            {"id", dataToken.Id}},
+            {"id", dataToken.Id}};
+      AddTypeInfo(dataToken.TypeId, dict);
+      return CreateNode(XmlStringConstants.PRIMITIVE, dict,
         ((XmlString)dataToken.SerialData).Data);
     }
@@ -149 +164 @@
     protected override string Format(MetaInfoEndToken metaInfoEndToken) {
       return CreateNodeEnd(XmlStringConstants.METAINFO);
+    }
+
+    private TypeToken lastTypeToken;
+    protected override string Format(TypeToken token) {
+      lastTypeToken = token;
+      return "";
+    }
+
+    private string FlushTypeToken() {
+      if (lastTypeToken == null)
+        return "";
+      try {
+        return CreateNode(XmlStringConstants.TYPE,
+          new Dictionary<string, object> {
+          {"id", lastTypeToken.Id},
+          {"typeName", lastTypeToken.TypeName },
+          {"serializer", lastTypeToken.Serializer }});
+      } finally {
+        lastTypeToken = null;
+      }
     }
 
@@ -182 +217 @@
     }
 
-
     public static void Serialize(object obj, string filename, Configuration config) {
       Serialize(obj, filename, config, false, 5);
@@ -192 +226 @@
         DateTime start = DateTime.Now;
         using (FileStream stream = File.Create(tempfile)) {
-          Serialize(obj, stream, config, includeAssemblies, compression);
+          Serializer serializer = new Serializer(obj, config);
+          serializer.InterleaveTypeInformation = false;
+          XmlGenerator generator = new XmlGenerator();
+          using (ZipOutputStream zipStream = new ZipOutputStream(stream)) {
+            zipStream.IsStreamOwner = false;
+            zipStream.SetLevel(compression);
+            zipStream.PutNextEntry(new ZipEntry("data.xml") { DateTime = DateTime.MinValue });
+            StreamWriter writer = new StreamWriter(zipStream);
+            foreach (ISerializationToken token in serializer) {
+              string line = generator.Format(token);
+              writer.Write(line);
+            }
+            writer.Flush();
+            zipStream.PutNextEntry(new ZipEntry("typecache.xml") { DateTime = DateTime.MinValue });
+            foreach (string line in generator.Format(serializer.TypeCache)) {
+              writer.Write(line);
+            }
+            writer.Flush();
+            if (includeAssemblies) {
+              foreach (string name in serializer.RequiredFiles) {
+                Uri uri = new Uri(name);
+                if (!uri.IsFile) {
+                  Logger.Warn("cannot read non-local files");
+                  continue;
+                }
+                zipStream.PutNextEntry(new ZipEntry(Path.GetFileName(uri.PathAndQuery)));
+                FileStream reader = File.OpenRead(uri.PathAndQuery);
+                byte[] buffer = new byte[1024 * 1024];
+                while (true) {
+                  int bytesRead = reader.Read(buffer, 0, 1024 * 1024);
+                  if (bytesRead == 0)
+                    break;
+                  zipStream.Write(buffer, 0, bytesRead);
+                }
+                writer.Flush();
+              }
+            }
+          }
         }
         Logger.Info(String.Format("serialization took {0} seconds with compression level {1}",
@@ -204 +275 @@
     }
 
+    public static void Serialize(object obj, Stream stream) {
+      Serialize(obj, stream, ConfigurationService.Instance.GetConfiguration(new XmlFormat()));
+    }
+
 
     public static void Serialize(object obj, Stream stream, Configuration config) {
       Serialize(obj, stream, config, false);
     }
-    
+
     public static void Serialize(object obj, Stream stream, Configuration config, bool includeAssemblies) {
-      Serialize(obj, stream, config, includeAssemblies, 9);
-    }
-
-    public static void Serialize(object obj, Stream stream, Configuration config, bool includeAssemblies, int compression) {      
+      Serialize(obj, stream, config, includeAssemblies, true);
+    }
+
+    public static void Serialize(object obj, Stream stream, Configuration config, bool includeAssemblies, bool interleaveTypeInfo) {      
       try {
-        Serializer serializer = new Serializer(obj, config);
-        XmlGenerator generator = new XmlGenerator();
-        using (ZipOutputStream zipStream = new ZipOutputStream(stream)) {
-          zipStream.IsStreamOwner = false;
-          zipStream.SetLevel(compression);
-          zipStream.PutNextEntry(new ZipEntry("data.xml") { DateTime = DateTime.MinValue });
-          StreamWriter writer = new StreamWriter(zipStream);
+        using (StreamWriter writer = new StreamWriter(new GZipStream(stream, CompressionMode.Compress))) {
+          Serializer serializer = new Serializer(obj, config);
+          serializer.InterleaveTypeInformation = true;
+          XmlGenerator generator = new XmlGenerator();
           foreach (ISerializationToken token in serializer) {
             string line = generator.Format(token);
@@ -227 +299 @@
           }
           writer.Flush();
-          zipStream.PutNextEntry(new ZipEntry("typecache.xml") { DateTime = DateTime.MinValue });
-          foreach (string line in generator.Format(serializer.TypeCache)) {
-            writer.Write(line);
-          }
-          writer.Flush();
-          if (includeAssemblies) {
-            foreach (string name in serializer.RequiredFiles) {
-              Uri uri = new Uri(name);
-              if (!uri.IsFile) {
-                Logger.Warn("cannot read non-local files");
-                continue;
-              }
-              zipStream.PutNextEntry(new ZipEntry(Path.GetFileName(uri.PathAndQuery)));
-              FileStream reader = File.OpenRead(uri.PathAndQuery);
-              byte[] buffer = new byte[1024 * 1024];
-              while (true) {
-                int bytesRead = reader.Read(buffer, 0, 1024 * 1024);
-                if (bytesRead == 0)
-                  break;
-                zipStream.Write(buffer, 0, bytesRead);
-              }
-              writer.Flush();
-            }
-          }
         }
       } catch (PersistenceException) {
@@ -256 +304 @@
       } catch (Exception e) {
         throw new PersistenceException("Unexpected exception during Serialization.", e);
-      }      
+      }
     }
   }

trunk/sources/HeuristicLab.Persistence/3.3/Default/Xml/XmlParser.cs

@@ -8 +8 @@
 using ICSharpCode.SharpZipLib.Zip;
 using HeuristicLab.Persistence.Core.Tokens;
+using System.IO.Compression;
 
 namespace HeuristicLab.Persistence.Default.Xml {
@@ -30 +31 @@
                      {XmlStringConstants.NULL, ParseNull},
                      {XmlStringConstants.METAINFO, ParseMetaInfo},
+                     {XmlStringConstants.TYPE, ParseTypeInfo},
                    };
     }
@@ -59 +61 @@
       string name = reader.GetAttribute("name");
       int typeId = int.Parse(reader.GetAttribute("typeId"));
+      string typeName = reader.GetAttribute("typeName");
+      string serializer = reader.GetAttribute("serializer");
+      if (typeName != null)
+        yield return new TypeToken(typeId, typeName, serializer);
       XmlReader inner = reader.ReadSubtree();
       inner.Read();
@@ -73 +79 @@
         id = int.Parse(idString);
       int typeId = int.Parse(reader.GetAttribute("typeId"));
+      string typeName = reader.GetAttribute("typeName");
+      string serializer = reader.GetAttribute("serializer");
+      if (typeName != null)
+        yield return new TypeToken(typeId, typeName, serializer);
       yield return new BeginToken(name, typeId, id);
       IEnumerator<ISerializationToken> iterator = GetEnumerator();
@@ -96 +106 @@
         yield return iterator.Current;
       yield return new MetaInfoEndToken();
+    }
+
+    private IEnumerator<ISerializationToken> ParseTypeInfo() {
+      yield return new TypeToken(
+        int.Parse(reader.GetAttribute("id")),
+        reader.GetAttribute("typeName"),
+        reader.GetAttribute("serializer"));
     }
 
@@ -129 +146 @@
 
     public static object Deserialize(Stream stream) {
-      using (ZipFile file = new ZipFile(stream)) {
-        return Deserialize(file);
+      try {
+        using (StreamReader reader = new StreamReader(new GZipStream(stream, CompressionMode.Decompress))) {
+          XmlParser parser = new XmlParser(reader);
+          Deserializer deserializer = new Deserializer(new TypeMapping[] { });
+          return deserializer.Deserialize(parser);
+        }
+      } catch (PersistenceException) {
+        throw;
+      } catch (Exception x) {
+        throw new PersistenceException("Unexpected exception during deserialization", x);
       }
     }
@@ -136 +161 @@
     private static object Deserialize(ZipFile zipFile) {
       try {
-        Deserializer deSerializer = new Deserializer(
-          ParseTypeCache(
-          new StreamReader(
-            zipFile.GetInputStream(zipFile.GetEntry("typecache.xml")))));
+        ZipEntry typecache = zipFile.GetEntry("typecache.xml");
+        if (typecache == null)
+          throw new PersistenceException("file does not contain typecache.xml");
+        Deserializer deSerializer = new Deserializer(ParseTypeCache(new StreamReader(zipFile.GetInputStream(typecache))));
+        ZipEntry data = zipFile.GetEntry("data.xml");
+        if (data == null)
+          throw new PersistenceException("file does not contain data.xml");
         XmlParser parser = new XmlParser(
-          new StreamReader(zipFile.GetInputStream(zipFile.GetEntry("data.xml"))));
+          new StreamReader(zipFile.GetInputStream(data)));
         object result = deSerializer.Deserialize(parser);
         zipFile.Close();
         return result;
-      } catch (PersistenceException e) {
+      } catch (PersistenceException) {
         throw;
       } catch (Exception e) {

trunk/sources/HeuristicLab.Persistence/3.3/HeuristicLab.Persistence-3.3.csproj

@@ -106 +106 @@
     <Compile Include="Core\FormatBase.cs" />
     <Compile Include="Auxiliary\TypeExtensions.cs" />
+    <Compile Include="Core\Tokens\TypeToken.cs" />
     <Compile Include="Core\TypeMapping.cs" />
     <Compile Include="Auxiliary\TypeName.cs" />

trunk/sources/HeuristicLab.Persistence/3.3/Tests/UseCases.cs

@@ -199 +199 @@
     [TestMethod]
     public void ComplexStorable() {
-      Root r = new Root();
-      r.intStack.Push(1);
-      r.intStack.Push(2);
-      r.intStack.Push(3);
-      r.selfReferences = new List<Root> { r, r };
-      r.c = new Custom { r = r };
-      r.dict.Add("one", 1);
-      r.dict.Add("two", 2);
-      r.dict.Add("three", 3);
-      r.myEnum = TestEnum.va1;
-      r.i = new[] { 7, 5, 6 };
-      r.s = "new value";
-      r.intArray = new ArrayList { 3, 2, 1 };
-      r.intList = new List<int> { 9, 8, 7 };
-      r.multiDimArray = new double[,] { { 5, 4, 3 }, { 1, 4, 6 } };
-      r.boolean = false;
-      r.dateTime = DateTime.Now;
-      r.kvp = new KeyValuePair<string, int>("string key", 321);
-      r.uninitialized = null;
+      Root r = InitializeComplexStorable();
       XmlGenerator.Serialize(r, tempFile);
       Root newR = (Root)XmlParser.Deserialize(tempFile);
+      CompareComplexStorables(r, newR);
+    }
+
+    private static void CompareComplexStorables(Root r, Root newR) {
       Assert.AreEqual(
         DebugStringGenerator.Serialize(r),
@@ -274 +260 @@
     }
 
+    private static Root InitializeComplexStorable() {
+      Root r = new Root();
+      r.intStack.Push(1);
+      r.intStack.Push(2);
+      r.intStack.Push(3);
+      r.selfReferences = new List<Root> { r, r };
+      r.c = new Custom { r = r };
+      r.dict.Add("one", 1);
+      r.dict.Add("two", 2);
+      r.dict.Add("three", 3);
+      r.myEnum = TestEnum.va1;
+      r.i = new[] { 7, 5, 6 };
+      r.s = "new value";
+      r.intArray = new ArrayList { 3, 2, 1 };
+      r.intList = new List<int> { 9, 8, 7 };
+      r.multiDimArray = new double[,] { { 5, 4, 3 }, { 1, 4, 6 } };
+      r.boolean = false;
+      r.dateTime = DateTime.Now;
+      r.kvp = new KeyValuePair<string, int>("string key", 321);
+      r.uninitialized = null;
+
+      return r;
+    }
+
     [TestMethod]
     public void SelfReferences() {
@@ -762 +772 @@
     }
 
+    [TestMethod]
+    public void TestStreaming() {
+      using (MemoryStream stream = new MemoryStream()) {
+        Root r = InitializeComplexStorable();
+        XmlGenerator.Serialize(r, stream);
+        using (MemoryStream stream2 = new MemoryStream(stream.ToArray())) {
+          Root newR = (Root)XmlParser.Deserialize(stream2);
+          CompareComplexStorables(r, newR);
+        }
+      }
+    }
+
     [ClassInitialize]
     public static void Initialize(TestContext testContext) {