Changeset 3005 for trunk/sources/HeuristicLab.Persistence/3.3/Default

Timestamp:

03/11/10 15:50:50 (15 years ago)

Author:

epitzer

Message:

add support for type information interleaving and subsequently true streaming (#548)

Location:

trunk/sources/HeuristicLab.Persistence/3.3/Default

Files:

• DebugString/DebugStringGenerator.cs (modified) (1 diff)
• Xml/XmlGenerator.cs (modified) (9 diffs)
• Xml/XmlParser.cs (modified) (7 diffs)

trunk/sources/HeuristicLab.Persistence/3.3/Default/DebugString/DebugStringGenerator.cs

@@ -97 +97 @@
     }
 
+    protected override string Format(TypeToken typeToken) {
+      return string.Empty;
+    }
+
     public static string Serialize(object o) {
       return Serialize(o, ConfigurationService.Instance.GetDefaultConfig(new DebugStringFormat()));

trunk/sources/HeuristicLab.Persistence/3.3/Default/Xml/XmlGenerator.cs

@@ -8 +8 @@
 using HeuristicLab.Tracing;
 using HeuristicLab.Persistence.Core.Tokens;
+using System.IO.Compression;
 
 namespace HeuristicLab.Persistence.Default.Xml {
@@ -109 +110 @@
 
     protected override string Format(BeginToken beginToken) {
-      return CreateNodeStart(
-        XmlStringConstants.COMPOSITE,
-        new Dictionary<string, object> {
+      var dict = new Dictionary<string, object> {
           {"name", beginToken.Name},
           {"typeId", beginToken.TypeId},
-          {"id", beginToken.Id}});
+          {"id", beginToken.Id}};
+      AddTypeInfo(beginToken.TypeId, dict);
+      return CreateNodeStart(XmlStringConstants.COMPOSITE, dict);
+        
+    }
+
+    private void AddTypeInfo(int typeId, Dictionary<string, object> dict) {
+      if (lastTypeToken != null) {
+        if (typeId == lastTypeToken.Id) {
+          dict.Add("typeName", lastTypeToken.TypeName);
+          dict.Add("serializer", lastTypeToken.Serializer);
+          lastTypeToken = null;
+        } else {
+          FlushTypeToken();
+        }
+      }
     }
 
@@ -122 +136 @@
 
     protected override string Format(PrimitiveToken dataToken) {
-      return CreateNode(XmlStringConstants.PRIMITIVE,
-        new Dictionary<string, object> {
+      var dict = new Dictionary<string, object> {
             {"typeId", dataToken.TypeId},
             {"name", dataToken.Name},
-            {"id", dataToken.Id}},
+            {"id", dataToken.Id}};
+      AddTypeInfo(dataToken.TypeId, dict);
+      return CreateNode(XmlStringConstants.PRIMITIVE, dict,
         ((XmlString)dataToken.SerialData).Data);
     }
@@ -149 +164 @@
     protected override string Format(MetaInfoEndToken metaInfoEndToken) {
       return CreateNodeEnd(XmlStringConstants.METAINFO);
+    }
+
+    private TypeToken lastTypeToken;
+    protected override string Format(TypeToken token) {
+      lastTypeToken = token;
+      return "";
+    }
+
+    private string FlushTypeToken() {
+      if (lastTypeToken == null)
+        return "";
+      try {
+        return CreateNode(XmlStringConstants.TYPE,
+          new Dictionary<string, object> {
+          {"id", lastTypeToken.Id},
+          {"typeName", lastTypeToken.TypeName },
+          {"serializer", lastTypeToken.Serializer }});
+      } finally {
+        lastTypeToken = null;
+      }
     }
 
@@ -182 +217 @@
     }
 
-
     public static void Serialize(object obj, string filename, Configuration config) {
       Serialize(obj, filename, config, false, 5);
@@ -192 +226 @@
         DateTime start = DateTime.Now;
         using (FileStream stream = File.Create(tempfile)) {
-          Serialize(obj, stream, config, includeAssemblies, compression);
+          Serializer serializer = new Serializer(obj, config);
+          serializer.InterleaveTypeInformation = false;
+          XmlGenerator generator = new XmlGenerator();
+          using (ZipOutputStream zipStream = new ZipOutputStream(stream)) {
+            zipStream.IsStreamOwner = false;
+            zipStream.SetLevel(compression);
+            zipStream.PutNextEntry(new ZipEntry("data.xml") { DateTime = DateTime.MinValue });
+            StreamWriter writer = new StreamWriter(zipStream);
+            foreach (ISerializationToken token in serializer) {
+              string line = generator.Format(token);
+              writer.Write(line);
+            }
+            writer.Flush();
+            zipStream.PutNextEntry(new ZipEntry("typecache.xml") { DateTime = DateTime.MinValue });
+            foreach (string line in generator.Format(serializer.TypeCache)) {
+              writer.Write(line);
+            }
+            writer.Flush();
+            if (includeAssemblies) {
+              foreach (string name in serializer.RequiredFiles) {
+                Uri uri = new Uri(name);
+                if (!uri.IsFile) {
+                  Logger.Warn("cannot read non-local files");
+                  continue;
+                }
+                zipStream.PutNextEntry(new ZipEntry(Path.GetFileName(uri.PathAndQuery)));
+                FileStream reader = File.OpenRead(uri.PathAndQuery);
+                byte[] buffer = new byte[1024 * 1024];
+                while (true) {
+                  int bytesRead = reader.Read(buffer, 0, 1024 * 1024);
+                  if (bytesRead == 0)
+                    break;
+                  zipStream.Write(buffer, 0, bytesRead);
+                }
+                writer.Flush();
+              }
+            }
+          }
         }
         Logger.Info(String.Format("serialization took {0} seconds with compression level {1}",
@@ -204 +275 @@
     }
 
+    public static void Serialize(object obj, Stream stream) {
+      Serialize(obj, stream, ConfigurationService.Instance.GetConfiguration(new XmlFormat()));
+    }
+
 
     public static void Serialize(object obj, Stream stream, Configuration config) {
       Serialize(obj, stream, config, false);
     }
-    
+
     public static void Serialize(object obj, Stream stream, Configuration config, bool includeAssemblies) {
-      Serialize(obj, stream, config, includeAssemblies, 9);
-    }
-
-    public static void Serialize(object obj, Stream stream, Configuration config, bool includeAssemblies, int compression) {      
+      Serialize(obj, stream, config, includeAssemblies, true);
+    }
+
+    public static void Serialize(object obj, Stream stream, Configuration config, bool includeAssemblies, bool interleaveTypeInfo) {      
       try {
-        Serializer serializer = new Serializer(obj, config);
-        XmlGenerator generator = new XmlGenerator();
-        using (ZipOutputStream zipStream = new ZipOutputStream(stream)) {
-          zipStream.IsStreamOwner = false;
-          zipStream.SetLevel(compression);
-          zipStream.PutNextEntry(new ZipEntry("data.xml") { DateTime = DateTime.MinValue });
-          StreamWriter writer = new StreamWriter(zipStream);
+        using (StreamWriter writer = new StreamWriter(new GZipStream(stream, CompressionMode.Compress))) {
+          Serializer serializer = new Serializer(obj, config);
+          serializer.InterleaveTypeInformation = true;
+          XmlGenerator generator = new XmlGenerator();
           foreach (ISerializationToken token in serializer) {
             string line = generator.Format(token);
@@ -227 +299 @@
           }
           writer.Flush();
-          zipStream.PutNextEntry(new ZipEntry("typecache.xml") { DateTime = DateTime.MinValue });
-          foreach (string line in generator.Format(serializer.TypeCache)) {
-            writer.Write(line);
-          }
-          writer.Flush();
-          if (includeAssemblies) {
-            foreach (string name in serializer.RequiredFiles) {
-              Uri uri = new Uri(name);
-              if (!uri.IsFile) {
-                Logger.Warn("cannot read non-local files");
-                continue;
-              }
-              zipStream.PutNextEntry(new ZipEntry(Path.GetFileName(uri.PathAndQuery)));
-              FileStream reader = File.OpenRead(uri.PathAndQuery);
-              byte[] buffer = new byte[1024 * 1024];
-              while (true) {
-                int bytesRead = reader.Read(buffer, 0, 1024 * 1024);
-                if (bytesRead == 0)
-                  break;
-                zipStream.Write(buffer, 0, bytesRead);
-              }
-              writer.Flush();
-            }
-          }
         }
       } catch (PersistenceException) {
@@ -256 +304 @@
       } catch (Exception e) {
         throw new PersistenceException("Unexpected exception during Serialization.", e);
-      }      
+      }
     }
   }

trunk/sources/HeuristicLab.Persistence/3.3/Default/Xml/XmlParser.cs

@@ -8 +8 @@
 using ICSharpCode.SharpZipLib.Zip;
 using HeuristicLab.Persistence.Core.Tokens;
+using System.IO.Compression;
 
 namespace HeuristicLab.Persistence.Default.Xml {
@@ -30 +31 @@
                      {XmlStringConstants.NULL, ParseNull},
                      {XmlStringConstants.METAINFO, ParseMetaInfo},
+                     {XmlStringConstants.TYPE, ParseTypeInfo},
                    };
     }
@@ -59 +61 @@
       string name = reader.GetAttribute("name");
       int typeId = int.Parse(reader.GetAttribute("typeId"));
+      string typeName = reader.GetAttribute("typeName");
+      string serializer = reader.GetAttribute("serializer");
+      if (typeName != null)
+        yield return new TypeToken(typeId, typeName, serializer);
       XmlReader inner = reader.ReadSubtree();
       inner.Read();
@@ -73 +79 @@
         id = int.Parse(idString);
       int typeId = int.Parse(reader.GetAttribute("typeId"));
+      string typeName = reader.GetAttribute("typeName");
+      string serializer = reader.GetAttribute("serializer");
+      if (typeName != null)
+        yield return new TypeToken(typeId, typeName, serializer);
       yield return new BeginToken(name, typeId, id);
       IEnumerator<ISerializationToken> iterator = GetEnumerator();
@@ -96 +106 @@
         yield return iterator.Current;
       yield return new MetaInfoEndToken();
+    }
+
+    private IEnumerator<ISerializationToken> ParseTypeInfo() {
+      yield return new TypeToken(
+        int.Parse(reader.GetAttribute("id")),
+        reader.GetAttribute("typeName"),
+        reader.GetAttribute("serializer"));
     }
 
@@ -129 +146 @@
 
     public static object Deserialize(Stream stream) {
-      using (ZipFile file = new ZipFile(stream)) {
-        return Deserialize(file);
+      try {
+        using (StreamReader reader = new StreamReader(new GZipStream(stream, CompressionMode.Decompress))) {
+          XmlParser parser = new XmlParser(reader);
+          Deserializer deserializer = new Deserializer(new TypeMapping[] { });
+          return deserializer.Deserialize(parser);
+        }
+      } catch (PersistenceException) {
+        throw;
+      } catch (Exception x) {
+        throw new PersistenceException("Unexpected exception during deserialization", x);
       }
     }
@@ -136 +161 @@
     private static object Deserialize(ZipFile zipFile) {
       try {
-        Deserializer deSerializer = new Deserializer(
-          ParseTypeCache(
-          new StreamReader(
-            zipFile.GetInputStream(zipFile.GetEntry("typecache.xml")))));
+        ZipEntry typecache = zipFile.GetEntry("typecache.xml");
+        if (typecache == null)
+          throw new PersistenceException("file does not contain typecache.xml");
+        Deserializer deSerializer = new Deserializer(ParseTypeCache(new StreamReader(zipFile.GetInputStream(typecache))));
+        ZipEntry data = zipFile.GetEntry("data.xml");
+        if (data == null)
+          throw new PersistenceException("file does not contain data.xml");
         XmlParser parser = new XmlParser(
-          new StreamReader(zipFile.GetInputStream(zipFile.GetEntry("data.xml"))));
+          new StreamReader(zipFile.GetInputStream(data)));
         object result = deSerializer.Deserialize(parser);
         zipFile.Close();
         return result;
-      } catch (PersistenceException e) {
+      } catch (PersistenceException) {
         throw;
       } catch (Exception e) {