source: branches/2965_CancelablePersistence/HeuristicLab.Persistence/3.3/Default/Xml/XmlGenerator.cs @ 16325

Last change on this file since 16325 was 16325, checked in by pfleck, 2 years ago

#2965

  • Added CancelationTokens for the Save and Serialize methods.
  • Fixed a potential temp-file-leak when replacing the old file with the new one after serialization.
File size: 19.8 KB
RevLine 
[3742]1#region License Information
2/* HeuristicLab
[15583]3 * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
[3742]4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
[4068]22using System;
[3742]23using System.Collections.Generic;
[4068]24using System.IO;
25using System.IO.Compression;
26using System.Linq;
[1454]27using System.Text;
[16325]28using System.Threading;
[4068]29using HeuristicLab.Persistence.Core;
30using HeuristicLab.Persistence.Core.Tokens;
[1454]31using HeuristicLab.Persistence.Interfaces;
[4068]32using HeuristicLab.Tracing;
[1454]33
[1615]34namespace HeuristicLab.Persistence.Default.Xml {
[1454]35
[3004]36
37  /// <summary>
38  /// Main entry point of persistence to XML. Use the static methods to serialize
39  /// to a file or to a stream.
40  /// </summary>
[1564]41  public class XmlGenerator : GeneratorBase<string> {
[1566]42
[3935]43    protected int depth;
44    protected int Depth {
[1570]45      get {
46        return depth;
47      }
48      set {
49        depth = value;
50        prefix = new string(' ', depth * 2);
51      }
52    }
[1454]53
[3935]54    protected string prefix;
[1570]55
56
[3016]57    /// <summary>
58    /// Initializes a new instance of the <see cref="XmlGenerator"/> class.
59    /// </summary>
[1454]60    public XmlGenerator() {
[1570]61      Depth = 0;
[1454]62    }
63
[16325]64    protected enum NodeType { Start, End, Inline };
[1454]65
[3937]66    protected static void AddXmlTagContent(StringBuilder sb, string name, Dictionary<string, string> attributes) {
[1566]67      sb.Append(name);
[1454]68      foreach (var attribute in attributes) {
69        if (attribute.Value != null && !string.IsNullOrEmpty(attribute.Value.ToString())) {
[1570]70          sb.Append(' ');
[1454]71          sb.Append(attribute.Key);
72          sb.Append("=\"");
73          sb.Append(attribute.Value);
74          sb.Append('"');
75        }
76      }
[1570]77    }
78
[3937]79    protected static int AttributeLength(Dictionary<string, string> attributes) {
80      return attributes
81        .Where(kvp => !string.IsNullOrEmpty(kvp.Key) && !string.IsNullOrEmpty(kvp.Value))
82        .Select(kvp => kvp.Key.Length + kvp.Value.Length + 4).Sum();
83    }
84
85    protected static void AddXmlStartTag(StringBuilder sb, string name, Dictionary<string, string> attributes) {
[1570]86      sb.Append('<');
87      AddXmlTagContent(sb, name, attributes);
88      sb.Append('>');
89    }
90
[3937]91    protected static void AddXmlInlineTag(StringBuilder sb, string name, Dictionary<string, string> attributes) {
[1570]92      sb.Append('<');
93      AddXmlTagContent(sb, name, attributes);
94      sb.Append("/>");
95    }
96
[3935]97    protected static void AddXmlEndTag(StringBuilder sb, string name) {
[1570]98      sb.Append("</");
99      sb.Append(name);
[1454]100      sb.Append(">");
[1570]101    }
102
[3937]103    protected string CreateNodeStart(string name, Dictionary<string, string> attributes) {
104      StringBuilder sb = new StringBuilder(prefix.Length + name.Length + 4
105        + AttributeLength(attributes));
[1570]106      sb.Append(prefix);
107      Depth += 1;
108      AddXmlStartTag(sb, name, attributes);
109      sb.Append("\r\n");
[1566]110      return sb.ToString();
[1454]111    }
112
[3937]113    private static Dictionary<string, string> emptyDict = new Dictionary<string, string>();
[3935]114    protected string CreateNodeStart(string name) {
[3937]115      return CreateNodeStart(name, emptyDict);
[1454]116    }
117
[3935]118    protected string CreateNodeEnd(string name) {
[1570]119      Depth -= 1;
[3937]120      StringBuilder sb = new StringBuilder(prefix.Length + name.Length + 5);
[1570]121      sb.Append(prefix);
122      AddXmlEndTag(sb, name);
123      sb.Append("\r\n");
124      return sb.ToString();
125    }
126
[3937]127    protected string CreateNode(string name, Dictionary<string, string> attributes) {
128      StringBuilder sb = new StringBuilder(prefix.Length + name.Length + 5
129        + AttributeLength(attributes));
[1570]130      sb.Append(prefix);
131      AddXmlInlineTag(sb, name, attributes);
132      sb.Append("\r\n");
133      return sb.ToString();
134    }
135
[3937]136    protected string CreateNode(string name, Dictionary<string, string> attributes, string content) {
[3944]137      StringBuilder sb = new StringBuilder(
138        prefix.Length + name.Length + AttributeLength(attributes) + 2
139        + content.Length + name.Length + 5);
[1570]140      sb.Append(prefix);
141      AddXmlStartTag(sb, name, attributes);
142      sb.Append(content);
143      sb.Append("</").Append(name).Append(">\r\n");
144      return sb.ToString();
145    }
146
[3036]147    /// <summary>
148    /// Formats the specified begin token.
149    /// </summary>
150    /// <param name="beginToken">The begin token.</param>
151    /// <returns>The token in serialized form.</returns>
[1566]152    protected override string Format(BeginToken beginToken) {
[3937]153      var dict = new Dictionary<string, string> {
[1570]154          {"name", beginToken.Name},
[3937]155          {"typeId", beginToken.TypeId.ToString()},
156          {"id", beginToken.Id.ToString()}};
[3005]157      AddTypeInfo(beginToken.TypeId, dict);
158      return CreateNodeStart(XmlStringConstants.COMPOSITE, dict);
[3944]159
[1454]160    }
161
[3937]162    protected void AddTypeInfo(int typeId, Dictionary<string, string> dict) {
[3005]163      if (lastTypeToken != null) {
164        if (typeId == lastTypeToken.Id) {
165          dict.Add("typeName", lastTypeToken.TypeName);
166          dict.Add("serializer", lastTypeToken.Serializer);
167          lastTypeToken = null;
168        } else {
169          FlushTypeToken();
170        }
171      }
172    }
173
[3036]174    /// <summary>
175    /// Formats the specified end token.
176    /// </summary>
177    /// <param name="endToken">The end token.</param>
178    /// <returns>The token in serialized form.</returns>
[1566]179    protected override string Format(EndToken endToken) {
[1612]180      return CreateNodeEnd(XmlStringConstants.COMPOSITE);
[1454]181    }
182
[3036]183    /// <summary>
184    /// Formats the specified data token.
185    /// </summary>
186    /// <param name="dataToken">The data token.</param>
187    /// <returns>The token in serialized form.</returns>
[1566]188    protected override string Format(PrimitiveToken dataToken) {
[3937]189      var dict = new Dictionary<string, string> {
190            {"typeId", dataToken.TypeId.ToString()},
[1454]191            {"name", dataToken.Name},
[3937]192            {"id", dataToken.Id.ToString()}};
[3005]193      AddTypeInfo(dataToken.TypeId, dict);
194      return CreateNode(XmlStringConstants.PRIMITIVE, dict,
[1570]195        ((XmlString)dataToken.SerialData).Data);
[1454]196    }
197
[3036]198    /// <summary>
199    /// Formats the specified ref token.
200    /// </summary>
201    /// <param name="refToken">The ref token.</param>
202    /// <returns>The token in serialized form.</returns>
[1566]203    protected override string Format(ReferenceToken refToken) {
[1612]204      return CreateNode(XmlStringConstants.REFERENCE,
[3937]205        new Dictionary<string, string> {
206          {"ref", refToken.Id.ToString()},
[1570]207          {"name", refToken.Name}});
[1454]208    }
209
[3036]210    /// <summary>
211    /// Formats the specified null ref token.
212    /// </summary>
213    /// <param name="nullRefToken">The null ref token.</param>
214    /// <returns>The token in serialized form.</returns>
[1566]215    protected override string Format(NullReferenceToken nullRefToken) {
[1612]216      return CreateNode(XmlStringConstants.NULL,
[3937]217        new Dictionary<string, string>{
[1570]218          {"name", nullRefToken.Name}});
[1454]219    }
220
[3036]221    /// <summary>
222    /// Formats the specified meta info begin token.
223    /// </summary>
224    /// <param name="metaInfoBeginToken">The meta info begin token.</param>
225    /// <returns>The token in serialized form.</returns>
[1553]226    protected override string Format(MetaInfoBeginToken metaInfoBeginToken) {
[1612]227      return CreateNodeStart(XmlStringConstants.METAINFO);
[1553]228    }
229
[3036]230    /// <summary>
231    /// Formats the specified meta info end token.
232    /// </summary>
233    /// <param name="metaInfoEndToken">The meta info end token.</param>
234    /// <returns>The token in serialized form.</returns>
[1553]235    protected override string Format(MetaInfoEndToken metaInfoEndToken) {
[1612]236      return CreateNodeEnd(XmlStringConstants.METAINFO);
[1553]237    }
238
[3935]239    protected TypeToken lastTypeToken;
[3036]240    /// <summary>
241    /// Formats the specified token.
242    /// </summary>
243    /// <param name="token">The token.</param>
244    /// <returns>The token in serialized form.</returns>
[3005]245    protected override string Format(TypeToken token) {
246      lastTypeToken = token;
247      return "";
248    }
249
[3935]250    protected string FlushTypeToken() {
[3005]251      if (lastTypeToken == null)
252        return "";
253      try {
254        return CreateNode(XmlStringConstants.TYPE,
[3937]255          new Dictionary<string, string> {
256          {"id", lastTypeToken.Id.ToString()},
[3005]257          {"typeName", lastTypeToken.TypeName },
258          {"serializer", lastTypeToken.Serializer }});
[16325]259      } finally {
[3005]260        lastTypeToken = null;
261      }
262    }
263
[3028]264    /// <summary>
265    /// Formats the specified type cache.
266    /// </summary>
267    /// <param name="typeCache">The type cache.</param>
268    /// <returns>An enumerable of formatted type cache tags.</returns>
[1454]269    public IEnumerable<string> Format(List<TypeMapping> typeCache) {
[1612]270      yield return CreateNodeStart(XmlStringConstants.TYPECACHE);
[1454]271      foreach (var mapping in typeCache)
[1570]272        yield return CreateNode(
[1612]273          XmlStringConstants.TYPE,
[1570]274          mapping.GetDict());
[1612]275      yield return CreateNodeEnd(XmlStringConstants.TYPECACHE);
[1454]276    }
277
[3004]278    /// <summary>
279    /// Serialize an object into a file.
[3016]280    /// The XML configuration is obtained from the <c>ConfigurationService</c>.
[3004]281    /// The file is actually a ZIP file.
282    /// Compression level is set to 5 and needed assemblies are not included.
[3036]283    /// </summary>
284    /// <param name="o">The object.</param>
285    /// <param name="filename">The filename.</param>
[16325]286    public static void Serialize(object o, string filename, CancellationToken cancellationToken = default(CancellationToken)) {
287      Serialize(o, filename, ConfigurationService.Instance.GetConfiguration(new XmlFormat()), false, CompressionLevel.Optimal, cancellationToken);
[1454]288    }
289
[3004]290    /// <summary>
291    /// Serialize an object into a file.
[3016]292    /// The XML configuration is obtained from the <c>ConfigurationService</c>.
[3004]293    /// Needed assemblies are not included.
294    /// </summary>
[3028]295    /// <param name="o">The object.</param>
296    /// <param name="filename">The filename.</param>
[3004]297    /// <param name="compression">ZIP file compression level</param>
[16325]298    public static void Serialize(object o, string filename, CompressionLevel compression, CancellationToken cancellationToken = default(CancellationToken)) {
299      Serialize(o, filename, ConfigurationService.Instance.GetConfiguration(new XmlFormat()), false, compression, cancellationToken);
[1892]300    }
301
[3028]302    /// <summary>
303    /// Serializes the specified object into a file.
304    /// Needed assemblies are not included, ZIP compression level is set to 5.
305    /// </summary>
306    /// <param name="obj">The object.</param>
307    /// <param name="filename">The filename.</param>
308    /// <param name="config">The configuration.</param>
[16325]309    public static void Serialize(object obj, string filename, Configuration config, CancellationToken cancellationToken = default(CancellationToken)) {
310      Serialize(obj, filename, config, false, CompressionLevel.Optimal, cancellationToken);
[1797]311    }
312
[16325]313    private static void Serialize(object obj, Stream stream, Configuration config, bool includeAssemblies, CompressionLevel compression, CancellationToken cancellationToken = default(CancellationToken)) {
[12455]314      Serializer serializer = new Serializer(obj, config);
[16325]315      Serialize(stream, includeAssemblies, compression, serializer, cancellationToken);
[12455]316    }
317
[16325]318    private static void Serialize(Stream stream, bool includeAssemblies, CompressionLevel compression, Serializer serializer, CancellationToken cancellationToken = default(CancellationToken)) {
[12455]319      try {
[16325]320        cancellationToken.ThrowIfCancellationRequested();
[12455]321        DateTime start = DateTime.Now;
322        serializer.InterleaveTypeInformation = false;
323        XmlGenerator generator = new XmlGenerator();
324        using (ZipArchive zipArchive = new ZipArchive(stream, ZipArchiveMode.Create)) {
325          ZipArchiveEntry entry = zipArchive.CreateEntry("data.xml", compression);
326          using (StreamWriter writer = new StreamWriter(entry.Open())) {
327            foreach (ISerializationToken token in serializer) {
[16325]328              cancellationToken.ThrowIfCancellationRequested();
[12455]329              string line = generator.Format(token);
330              writer.Write(line);
331            }
332          }
333          entry = zipArchive.CreateEntry("typecache.xml", compression);
334          using (StreamWriter writer = new StreamWriter(entry.Open())) {
335            foreach (string line in generator.Format(serializer.TypeCache)) {
336              writer.Write(line);
337            }
338          }
339          if (includeAssemblies) {
340            foreach (string name in serializer.RequiredFiles) {
[16325]341              cancellationToken.ThrowIfCancellationRequested();
[12455]342              Uri uri = new Uri(name);
343              if (!uri.IsFile) {
344                Logger.Warn("cannot read non-local files");
345                continue;
346              }
347              entry = zipArchive.CreateEntry(Path.GetFileName(uri.PathAndQuery), compression);
348              using (BinaryWriter bw = new BinaryWriter(entry.Open())) {
349                using (FileStream reader = File.OpenRead(uri.PathAndQuery)) {
350                  byte[] buffer = new byte[1024 * 1024];
351                  while (true) {
[16325]352                    cancellationToken.ThrowIfCancellationRequested();
[12455]353                    int bytesRead = reader.Read(buffer, 0, 1024 * 1024);
354                    if (bytesRead == 0)
355                      break;
356                    bw.Write(buffer, 0, bytesRead);
357                  }
358                }
359              }
360            }
361          }
362        }
363        Logger.Info(String.Format("serialization took {0} seconds with compression level {1}",
364          (DateTime.Now - start).TotalSeconds, compression));
[16325]365      } catch (Exception) {
[12455]366        Logger.Warn("Exception caught, no data has been serialized.");
367        throw;
368      }
369    }
370
[3028]371    /// <summary>
372    /// Serializes the specified object into a file.
373    /// </summary>
374    /// <param name="obj">The object.</param>
375    /// <param name="filename">The filename.</param>
376    /// <param name="config">The configuration.</param>
377    /// <param name="includeAssemblies">if set to <c>true</c> include needed assemblies.</param>
378    /// <param name="compression">The ZIP compression level.</param>
[16325]379    public static void Serialize(object obj, string filename, Configuration config, bool includeAssemblies, CompressionLevel compression, CancellationToken cancellationToken = default(CancellationToken)) {
380      string tempfile = null;
[1625]381      try {
[16325]382        tempfile = Path.GetTempFileName();
[12455]383
[2037]384        using (FileStream stream = File.Create(tempfile)) {
[16325]385          Serialize(obj, stream, config, includeAssemblies, compression, cancellationToken);
[2037]386        }
[16325]387        // copy only if needed
[1733]388        File.Copy(tempfile, filename, true);
[16325]389      } catch (Exception) {
[1733]390        Logger.Warn("Exception caught, no data has been written.");
391        throw;
[16325]392      } finally {
393        if (tempfile != null && File.Exists(tempfile))
394          File.Delete(tempfile);
[1733]395      }
396    }
397
[3028]398    /// <summary>
399    /// Serializes the specified object into a stream using the <see cref="XmlFormat"/>
400    /// obtained from the <see cref="ConfigurationService"/>.
401    /// </summary>
402    /// <param name="obj">The object.</param>
403    /// <param name="stream">The stream.</param>
[12638]404    /// <param name="compressionType">Type of compression, default is GZip.</param>
[16325]405    public static void Serialize(object obj, Stream stream, CompressionType compressionType = CompressionType.GZip, CancellationToken cancellationToken = default(CancellationToken)) {
406      Serialize(obj, stream, ConfigurationService.Instance.GetConfiguration(new XmlFormat()), compressionType, cancellationToken);
[3005]407    }
[1797]408
[3005]409
[3028]410    /// <summary>
411    /// Serializes the specified object into a stream.
412    /// </summary>
413    /// <param name="obj">The object.</param>
414    /// <param name="stream">The stream.</param>
415    /// <param name="config">The configuration.</param>
[12638]416    /// <param name="compressionType">Type of compression, default is GZip.</param>
[16325]417    public static void Serialize(object obj, Stream stream, Configuration config, CompressionType compressionType = CompressionType.GZip, CancellationToken cancellationToken = default(CancellationToken)) {
418      Serialize(obj, stream, config, false, compressionType, cancellationToken);
[1797]419    }
[3005]420
[3028]421    /// <summary>
422    /// Serializes the specified object into a stream.
423    /// </summary>
424    /// <param name="obj">The object.</param>
425    /// <param name="stream">The stream.</param>
426    /// <param name="config">The configuration.</param>
427    /// <param name="includeAssemblies">if set to <c>true</c> include need assemblies.</param>
[12638]428    /// <param name="compressionType">Type of compression, default is GZip.</param>
[16325]429    public static void Serialize(object obj, Stream stream, Configuration config, bool includeAssemblies, CompressionType compressionType = CompressionType.GZip, CancellationToken cancellationToken = default(CancellationToken)) {
[1733]430      try {
[5403]431        Serializer serializer = new Serializer(obj, config);
[12638]432        if (compressionType == CompressionType.Zip) {
[16325]433          Serialize(obj, stream, config, includeAssemblies, CompressionLevel.Optimal, cancellationToken);
[12455]434        } else {
[16325]435          Serialize(stream, serializer, cancellationToken);
[12455]436        }
[16325]437      } catch (PersistenceException) {
[5403]438        throw;
[16325]439      } catch (Exception e) {
[5403]440        throw new PersistenceException("Unexpected exception during Serialization.", e);
[4068]441      }
[5403]442    }
443
444    /// <summary>
445    /// Serializes the specified object into a stream.
446    /// </summary>
447    /// <param name="obj">The object.</param>
448    /// <param name="stream">The stream.</param>
449    /// <param name="config">The configuration.</param>
450    /// <param name="includeAssemblies">if set to <c>true</c> include need assemblies.</param>
451    /// <param name="types">The list of all serialized types.</param>
[12638]452    /// <param name="compressionType">Type of compression, default is GZip.</param>
453    public static void Serialize(object obj, Stream stream, Configuration config, bool includeAssemblies, out IEnumerable<Type> types,
[16325]454                                 CompressionType compressionType = CompressionType.GZip, CancellationToken cancellationToken = default(CancellationToken)) {
[5403]455      try {
456        Serializer serializer = new Serializer(obj, config);
[12638]457        if (compressionType == CompressionType.Zip) {
[16325]458          Serialize(stream, includeAssemblies, CompressionLevel.Optimal, serializer, cancellationToken);
[12455]459        } else {
[16325]460          Serialize(stream, serializer, cancellationToken);
[12455]461        }
[5403]462        types = serializer.SerializedTypes;
[16325]463      } catch (PersistenceException) {
[1625]464        throw;
[16325]465      } catch (Exception e) {
[1625]466        throw new PersistenceException("Unexpected exception during Serialization.", e);
[3005]467      }
[1454]468    }
[5403]469
[16325]470    private static void Serialize(Stream stream, Serializer serializer, CancellationToken cancellationToken = default(CancellationToken)) {
471      cancellationToken.ThrowIfCancellationRequested();
[5403]472      using (StreamWriter writer = new StreamWriter(new GZipStream(stream, CompressionMode.Compress))) {
473        serializer.InterleaveTypeInformation = true;
474        XmlGenerator generator = new XmlGenerator();
475        foreach (ISerializationToken token in serializer) {
[16325]476          cancellationToken.ThrowIfCancellationRequested();
[5403]477          string line = generator.Format(token);
478          writer.Write(line);
479        }
480        writer.Flush();
481      }
482    }
[1454]483  }
484}
Note: See TracBrowser for help on using the repository browser.