Changeset 3005 for trunk/sources/HeuristicLab.Persistence/3.3/Default/Xml/XmlGenerator.cs

Timestamp:

03/11/10 15:50:50 (15 years ago)

Author:

epitzer

Message:

add support for type information interleaving and subsequently true streaming (#548)

File:

• trunk/sources/HeuristicLab.Persistence/3.3/Default/Xml/XmlGenerator.cs (modified) (9 diffs)

trunk/sources/HeuristicLab.Persistence/3.3/Default/Xml/XmlGenerator.cs

@@ -8 +8 @@
 using HeuristicLab.Tracing;
 using HeuristicLab.Persistence.Core.Tokens;
+using System.IO.Compression;
 
 namespace HeuristicLab.Persistence.Default.Xml {
@@ -109 +110 @@
 
     protected override string Format(BeginToken beginToken) {
-      return CreateNodeStart(
-        XmlStringConstants.COMPOSITE,
-        new Dictionary<string, object> {
+      var dict = new Dictionary<string, object> {
           {"name", beginToken.Name},
           {"typeId", beginToken.TypeId},
-          {"id", beginToken.Id}});
+          {"id", beginToken.Id}};
+      AddTypeInfo(beginToken.TypeId, dict);
+      return CreateNodeStart(XmlStringConstants.COMPOSITE, dict);
+        
+    }
+
+    private void AddTypeInfo(int typeId, Dictionary<string, object> dict) {
+      if (lastTypeToken != null) {
+        if (typeId == lastTypeToken.Id) {
+          dict.Add("typeName", lastTypeToken.TypeName);
+          dict.Add("serializer", lastTypeToken.Serializer);
+          lastTypeToken = null;
+        } else {
+          FlushTypeToken();
+        }
+      }
     }
 
@@ -122 +136 @@
 
     protected override string Format(PrimitiveToken dataToken) {
-      return CreateNode(XmlStringConstants.PRIMITIVE,
-        new Dictionary<string, object> {
+      var dict = new Dictionary<string, object> {
             {"typeId", dataToken.TypeId},
             {"name", dataToken.Name},
-            {"id", dataToken.Id}},
+            {"id", dataToken.Id}};
+      AddTypeInfo(dataToken.TypeId, dict);
+      return CreateNode(XmlStringConstants.PRIMITIVE, dict,
         ((XmlString)dataToken.SerialData).Data);
     }
@@ -149 +164 @@
     protected override string Format(MetaInfoEndToken metaInfoEndToken) {
       return CreateNodeEnd(XmlStringConstants.METAINFO);
+    }
+
+    private TypeToken lastTypeToken;
+    protected override string Format(TypeToken token) {
+      lastTypeToken = token;
+      return "";
+    }
+
+    private string FlushTypeToken() {
+      if (lastTypeToken == null)
+        return "";
+      try {
+        return CreateNode(XmlStringConstants.TYPE,
+          new Dictionary<string, object> {
+          {"id", lastTypeToken.Id},
+          {"typeName", lastTypeToken.TypeName },
+          {"serializer", lastTypeToken.Serializer }});
+      } finally {
+        lastTypeToken = null;
+      }
     }
 
@@ -182 +217 @@
     }
 
-
     public static void Serialize(object obj, string filename, Configuration config) {
       Serialize(obj, filename, config, false, 5);
@@ -192 +226 @@
         DateTime start = DateTime.Now;
         using (FileStream stream = File.Create(tempfile)) {
-          Serialize(obj, stream, config, includeAssemblies, compression);
+          Serializer serializer = new Serializer(obj, config);
+          serializer.InterleaveTypeInformation = false;
+          XmlGenerator generator = new XmlGenerator();
+          using (ZipOutputStream zipStream = new ZipOutputStream(stream)) {
+            zipStream.IsStreamOwner = false;
+            zipStream.SetLevel(compression);
+            zipStream.PutNextEntry(new ZipEntry("data.xml") { DateTime = DateTime.MinValue });
+            StreamWriter writer = new StreamWriter(zipStream);
+            foreach (ISerializationToken token in serializer) {
+              string line = generator.Format(token);
+              writer.Write(line);
+            }
+            writer.Flush();
+            zipStream.PutNextEntry(new ZipEntry("typecache.xml") { DateTime = DateTime.MinValue });
+            foreach (string line in generator.Format(serializer.TypeCache)) {
+              writer.Write(line);
+            }
+            writer.Flush();
+            if (includeAssemblies) {
+              foreach (string name in serializer.RequiredFiles) {
+                Uri uri = new Uri(name);
+                if (!uri.IsFile) {
+                  Logger.Warn("cannot read non-local files");
+                  continue;
+                }
+                zipStream.PutNextEntry(new ZipEntry(Path.GetFileName(uri.PathAndQuery)));
+                FileStream reader = File.OpenRead(uri.PathAndQuery);
+                byte[] buffer = new byte[1024 * 1024];
+                while (true) {
+                  int bytesRead = reader.Read(buffer, 0, 1024 * 1024);
+                  if (bytesRead == 0)
+                    break;
+                  zipStream.Write(buffer, 0, bytesRead);
+                }
+                writer.Flush();
+              }
+            }
+          }
         }
         Logger.Info(String.Format("serialization took {0} seconds with compression level {1}",
@@ -204 +275 @@
     }
 
+    public static void Serialize(object obj, Stream stream) {
+      Serialize(obj, stream, ConfigurationService.Instance.GetConfiguration(new XmlFormat()));
+    }
+
 
     public static void Serialize(object obj, Stream stream, Configuration config) {
       Serialize(obj, stream, config, false);
     }
-    
+
     public static void Serialize(object obj, Stream stream, Configuration config, bool includeAssemblies) {
-      Serialize(obj, stream, config, includeAssemblies, 9);
-    }
-
-    public static void Serialize(object obj, Stream stream, Configuration config, bool includeAssemblies, int compression) {      
+      Serialize(obj, stream, config, includeAssemblies, true);
+    }
+
+    public static void Serialize(object obj, Stream stream, Configuration config, bool includeAssemblies, bool interleaveTypeInfo) {      
       try {
-        Serializer serializer = new Serializer(obj, config);
-        XmlGenerator generator = new XmlGenerator();
-        using (ZipOutputStream zipStream = new ZipOutputStream(stream)) {
-          zipStream.IsStreamOwner = false;
-          zipStream.SetLevel(compression);
-          zipStream.PutNextEntry(new ZipEntry("data.xml") { DateTime = DateTime.MinValue });
-          StreamWriter writer = new StreamWriter(zipStream);
+        using (StreamWriter writer = new StreamWriter(new GZipStream(stream, CompressionMode.Compress))) {
+          Serializer serializer = new Serializer(obj, config);
+          serializer.InterleaveTypeInformation = true;
+          XmlGenerator generator = new XmlGenerator();
           foreach (ISerializationToken token in serializer) {
             string line = generator.Format(token);
@@ -227 +299 @@
           }
           writer.Flush();
-          zipStream.PutNextEntry(new ZipEntry("typecache.xml") { DateTime = DateTime.MinValue });
-          foreach (string line in generator.Format(serializer.TypeCache)) {
-            writer.Write(line);
-          }
-          writer.Flush();
-          if (includeAssemblies) {
-            foreach (string name in serializer.RequiredFiles) {
-              Uri uri = new Uri(name);
-              if (!uri.IsFile) {
-                Logger.Warn("cannot read non-local files");
-                continue;
-              }
-              zipStream.PutNextEntry(new ZipEntry(Path.GetFileName(uri.PathAndQuery)));
-              FileStream reader = File.OpenRead(uri.PathAndQuery);
-              byte[] buffer = new byte[1024 * 1024];
-              while (true) {
-                int bytesRead = reader.Read(buffer, 0, 1024 * 1024);
-                if (bytesRead == 0)
-                  break;
-                zipStream.Write(buffer, 0, bytesRead);
-              }
-              writer.Flush();
-            }
-          }
         }
       } catch (PersistenceException) {
@@ -256 +304 @@
       } catch (Exception e) {
         throw new PersistenceException("Unexpected exception during Serialization.", e);
-      }      
+      }
     }
   }