Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/HeuristicLab.Persistence/3.3/Default/Xml/XmlGenerator.cs @ 16440

Last change on this file since 16440 was 16440, checked in by pfleck, 5 years ago

#2965 merged branch to trunk

File size: 19.8 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.IO;
25using System.IO.Compression;
26using System.Linq;
27using System.Text;
28using System.Threading;
29using HeuristicLab.Persistence.Core;
30using HeuristicLab.Persistence.Core.Tokens;
31using HeuristicLab.Persistence.Interfaces;
32using HeuristicLab.Tracing;
33
34namespace HeuristicLab.Persistence.Default.Xml {
35
36
37  /// <summary>
38  /// Main entry point of persistence to XML. Use the static methods to serialize
39  /// to a file or to a stream.
40  /// </summary>
41  public class XmlGenerator : GeneratorBase<string> {
42
43    protected int depth;
44    protected int Depth {
45      get {
46        return depth;
47      }
48      set {
49        depth = value;
50        prefix = new string(' ', depth * 2);
51      }
52    }
53
54    protected string prefix;
55
56
57    /// <summary>
58    /// Initializes a new instance of the <see cref="XmlGenerator"/> class.
59    /// </summary>
60    public XmlGenerator() {
61      Depth = 0;
62    }
63
64    protected enum NodeType { Start, End, Inline };
65
66    protected static void AddXmlTagContent(StringBuilder sb, string name, Dictionary<string, string> attributes) {
67      sb.Append(name);
68      foreach (var attribute in attributes) {
69        if (attribute.Value != null && !string.IsNullOrEmpty(attribute.Value.ToString())) {
70          sb.Append(' ');
71          sb.Append(attribute.Key);
72          sb.Append("=\"");
73          sb.Append(attribute.Value);
74          sb.Append('"');
75        }
76      }
77    }
78
79    protected static int AttributeLength(Dictionary<string, string> attributes) {
80      return attributes
81        .Where(kvp => !string.IsNullOrEmpty(kvp.Key) && !string.IsNullOrEmpty(kvp.Value))
82        .Select(kvp => kvp.Key.Length + kvp.Value.Length + 4).Sum();
83    }
84
85    protected static void AddXmlStartTag(StringBuilder sb, string name, Dictionary<string, string> attributes) {
86      sb.Append('<');
87      AddXmlTagContent(sb, name, attributes);
88      sb.Append('>');
89    }
90
91    protected static void AddXmlInlineTag(StringBuilder sb, string name, Dictionary<string, string> attributes) {
92      sb.Append('<');
93      AddXmlTagContent(sb, name, attributes);
94      sb.Append("/>");
95    }
96
97    protected static void AddXmlEndTag(StringBuilder sb, string name) {
98      sb.Append("</");
99      sb.Append(name);
100      sb.Append(">");
101    }
102
103    protected string CreateNodeStart(string name, Dictionary<string, string> attributes) {
104      StringBuilder sb = new StringBuilder(prefix.Length + name.Length + 4
105        + AttributeLength(attributes));
106      sb.Append(prefix);
107      Depth += 1;
108      AddXmlStartTag(sb, name, attributes);
109      sb.Append("\r\n");
110      return sb.ToString();
111    }
112
113    private static Dictionary<string, string> emptyDict = new Dictionary<string, string>();
114    protected string CreateNodeStart(string name) {
115      return CreateNodeStart(name, emptyDict);
116    }
117
118    protected string CreateNodeEnd(string name) {
119      Depth -= 1;
120      StringBuilder sb = new StringBuilder(prefix.Length + name.Length + 5);
121      sb.Append(prefix);
122      AddXmlEndTag(sb, name);
123      sb.Append("\r\n");
124      return sb.ToString();
125    }
126
127    protected string CreateNode(string name, Dictionary<string, string> attributes) {
128      StringBuilder sb = new StringBuilder(prefix.Length + name.Length + 5
129        + AttributeLength(attributes));
130      sb.Append(prefix);
131      AddXmlInlineTag(sb, name, attributes);
132      sb.Append("\r\n");
133      return sb.ToString();
134    }
135
136    protected string CreateNode(string name, Dictionary<string, string> attributes, string content) {
137      StringBuilder sb = new StringBuilder(
138        prefix.Length + name.Length + AttributeLength(attributes) + 2
139        + content.Length + name.Length + 5);
140      sb.Append(prefix);
141      AddXmlStartTag(sb, name, attributes);
142      sb.Append(content);
143      sb.Append("</").Append(name).Append(">\r\n");
144      return sb.ToString();
145    }
146
147    /// <summary>
148    /// Formats the specified begin token.
149    /// </summary>
150    /// <param name="beginToken">The begin token.</param>
151    /// <returns>The token in serialized form.</returns>
152    protected override string Format(BeginToken beginToken) {
153      var dict = new Dictionary<string, string> {
154          {"name", beginToken.Name},
155          {"typeId", beginToken.TypeId.ToString()},
156          {"id", beginToken.Id.ToString()}};
157      AddTypeInfo(beginToken.TypeId, dict);
158      return CreateNodeStart(XmlStringConstants.COMPOSITE, dict);
159
160    }
161
162    protected void AddTypeInfo(int typeId, Dictionary<string, string> dict) {
163      if (lastTypeToken != null) {
164        if (typeId == lastTypeToken.Id) {
165          dict.Add("typeName", lastTypeToken.TypeName);
166          dict.Add("serializer", lastTypeToken.Serializer);
167          lastTypeToken = null;
168        } else {
169          FlushTypeToken();
170        }
171      }
172    }
173
174    /// <summary>
175    /// Formats the specified end token.
176    /// </summary>
177    /// <param name="endToken">The end token.</param>
178    /// <returns>The token in serialized form.</returns>
179    protected override string Format(EndToken endToken) {
180      return CreateNodeEnd(XmlStringConstants.COMPOSITE);
181    }
182
183    /// <summary>
184    /// Formats the specified data token.
185    /// </summary>
186    /// <param name="dataToken">The data token.</param>
187    /// <returns>The token in serialized form.</returns>
188    protected override string Format(PrimitiveToken dataToken) {
189      var dict = new Dictionary<string, string> {
190            {"typeId", dataToken.TypeId.ToString()},
191            {"name", dataToken.Name},
192            {"id", dataToken.Id.ToString()}};
193      AddTypeInfo(dataToken.TypeId, dict);
194      return CreateNode(XmlStringConstants.PRIMITIVE, dict,
195        ((XmlString)dataToken.SerialData).Data);
196    }
197
198    /// <summary>
199    /// Formats the specified ref token.
200    /// </summary>
201    /// <param name="refToken">The ref token.</param>
202    /// <returns>The token in serialized form.</returns>
203    protected override string Format(ReferenceToken refToken) {
204      return CreateNode(XmlStringConstants.REFERENCE,
205        new Dictionary<string, string> {
206          {"ref", refToken.Id.ToString()},
207          {"name", refToken.Name}});
208    }
209
210    /// <summary>
211    /// Formats the specified null ref token.
212    /// </summary>
213    /// <param name="nullRefToken">The null ref token.</param>
214    /// <returns>The token in serialized form.</returns>
215    protected override string Format(NullReferenceToken nullRefToken) {
216      return CreateNode(XmlStringConstants.NULL,
217        new Dictionary<string, string>{
218          {"name", nullRefToken.Name}});
219    }
220
221    /// <summary>
222    /// Formats the specified meta info begin token.
223    /// </summary>
224    /// <param name="metaInfoBeginToken">The meta info begin token.</param>
225    /// <returns>The token in serialized form.</returns>
226    protected override string Format(MetaInfoBeginToken metaInfoBeginToken) {
227      return CreateNodeStart(XmlStringConstants.METAINFO);
228    }
229
230    /// <summary>
231    /// Formats the specified meta info end token.
232    /// </summary>
233    /// <param name="metaInfoEndToken">The meta info end token.</param>
234    /// <returns>The token in serialized form.</returns>
235    protected override string Format(MetaInfoEndToken metaInfoEndToken) {
236      return CreateNodeEnd(XmlStringConstants.METAINFO);
237    }
238
239    protected TypeToken lastTypeToken;
240    /// <summary>
241    /// Formats the specified token.
242    /// </summary>
243    /// <param name="token">The token.</param>
244    /// <returns>The token in serialized form.</returns>
245    protected override string Format(TypeToken token) {
246      lastTypeToken = token;
247      return "";
248    }
249
250    protected string FlushTypeToken() {
251      if (lastTypeToken == null)
252        return "";
253      try {
254        return CreateNode(XmlStringConstants.TYPE,
255          new Dictionary<string, string> {
256          {"id", lastTypeToken.Id.ToString()},
257          {"typeName", lastTypeToken.TypeName },
258          {"serializer", lastTypeToken.Serializer }});
259      } finally {
260        lastTypeToken = null;
261      }
262    }
263
264    /// <summary>
265    /// Formats the specified type cache.
266    /// </summary>
267    /// <param name="typeCache">The type cache.</param>
268    /// <returns>An enumerable of formatted type cache tags.</returns>
269    public IEnumerable<string> Format(List<TypeMapping> typeCache) {
270      yield return CreateNodeStart(XmlStringConstants.TYPECACHE);
271      foreach (var mapping in typeCache)
272        yield return CreateNode(
273          XmlStringConstants.TYPE,
274          mapping.GetDict());
275      yield return CreateNodeEnd(XmlStringConstants.TYPECACHE);
276    }
277
278    /// <summary>
279    /// Serialize an object into a file.
280    /// The XML configuration is obtained from the <c>ConfigurationService</c>.
281    /// The file is actually a ZIP file.
282    /// Compression level is set to 5 and needed assemblies are not included.
283    /// </summary>
284    /// <param name="o">The object.</param>
285    /// <param name="filename">The filename.</param>
286    public static void Serialize(object o, string filename, CancellationToken cancellationToken = default(CancellationToken)) {
287      Serialize(o, filename, ConfigurationService.Instance.GetConfiguration(new XmlFormat()), false, CompressionLevel.Optimal, cancellationToken);
288    }
289
290    /// <summary>
291    /// Serialize an object into a file.
292    /// The XML configuration is obtained from the <c>ConfigurationService</c>.
293    /// Needed assemblies are not included.
294    /// </summary>
295    /// <param name="o">The object.</param>
296    /// <param name="filename">The filename.</param>
297    /// <param name="compression">ZIP file compression level</param>
298    public static void Serialize(object o, string filename, CompressionLevel compression, CancellationToken cancellationToken = default(CancellationToken)) {
299      Serialize(o, filename, ConfigurationService.Instance.GetConfiguration(new XmlFormat()), false, compression, cancellationToken);
300    }
301
302    /// <summary>
303    /// Serializes the specified object into a file.
304    /// Needed assemblies are not included, ZIP compression level is set to 5.
305    /// </summary>
306    /// <param name="obj">The object.</param>
307    /// <param name="filename">The filename.</param>
308    /// <param name="config">The configuration.</param>
309    public static void Serialize(object obj, string filename, Configuration config, CancellationToken cancellationToken = default(CancellationToken)) {
310      Serialize(obj, filename, config, false, CompressionLevel.Optimal, cancellationToken);
311    }
312
313    private static void Serialize(object obj, Stream stream, Configuration config, bool includeAssemblies, CompressionLevel compression, CancellationToken cancellationToken = default(CancellationToken)) {
314      Serializer serializer = new Serializer(obj, config);
315      Serialize(stream, includeAssemblies, compression, serializer, cancellationToken);
316    }
317
318    private static void Serialize(Stream stream, bool includeAssemblies, CompressionLevel compression, Serializer serializer, CancellationToken cancellationToken = default(CancellationToken)) {
319      try {
320        cancellationToken.ThrowIfCancellationRequested();
321        DateTime start = DateTime.Now;
322        serializer.InterleaveTypeInformation = false;
323        XmlGenerator generator = new XmlGenerator();
324        using (ZipArchive zipArchive = new ZipArchive(stream, ZipArchiveMode.Create)) {
325          ZipArchiveEntry entry = zipArchive.CreateEntry("data.xml", compression);
326          using (StreamWriter writer = new StreamWriter(entry.Open())) {
327            foreach (ISerializationToken token in serializer) {
328              cancellationToken.ThrowIfCancellationRequested();
329              string line = generator.Format(token);
330              writer.Write(line);
331            }
332          }
333          entry = zipArchive.CreateEntry("typecache.xml", compression);
334          using (StreamWriter writer = new StreamWriter(entry.Open())) {
335            foreach (string line in generator.Format(serializer.TypeCache)) {
336              writer.Write(line);
337            }
338          }
339          if (includeAssemblies) {
340            foreach (string name in serializer.RequiredFiles) {
341              cancellationToken.ThrowIfCancellationRequested();
342              Uri uri = new Uri(name);
343              if (!uri.IsFile) {
344                Logger.Warn("cannot read non-local files");
345                continue;
346              }
347              entry = zipArchive.CreateEntry(Path.GetFileName(uri.PathAndQuery), compression);
348              using (BinaryWriter bw = new BinaryWriter(entry.Open())) {
349                using (FileStream reader = File.OpenRead(uri.PathAndQuery)) {
350                  byte[] buffer = new byte[1024 * 1024];
351                  while (true) {
352                    cancellationToken.ThrowIfCancellationRequested();
353                    int bytesRead = reader.Read(buffer, 0, 1024 * 1024);
354                    if (bytesRead == 0)
355                      break;
356                    bw.Write(buffer, 0, bytesRead);
357                  }
358                }
359              }
360            }
361          }
362        }
363        Logger.Info(String.Format("serialization took {0} seconds with compression level {1}",
364          (DateTime.Now - start).TotalSeconds, compression));
365      } catch (Exception) {
366        Logger.Warn("Exception caught, no data has been serialized.");
367        throw;
368      }
369    }
370
371    /// <summary>
372    /// Serializes the specified object into a file.
373    /// </summary>
374    /// <param name="obj">The object.</param>
375    /// <param name="filename">The filename.</param>
376    /// <param name="config">The configuration.</param>
377    /// <param name="includeAssemblies">if set to <c>true</c> include needed assemblies.</param>
378    /// <param name="compression">The ZIP compression level.</param>
379    public static void Serialize(object obj, string filename, Configuration config, bool includeAssemblies, CompressionLevel compression, CancellationToken cancellationToken = default(CancellationToken)) {
380      string tempfile = null;
381      try {
382        tempfile = Path.GetTempFileName();
383
384        using (FileStream stream = File.Create(tempfile)) {
385          Serialize(obj, stream, config, includeAssemblies, compression, cancellationToken);
386        }
387        // copy only if needed
388        File.Copy(tempfile, filename, true);
389      } catch (Exception) {
390        Logger.Warn("Exception caught, no data has been written.");
391        throw;
392      } finally {
393        if (tempfile != null && File.Exists(tempfile))
394          File.Delete(tempfile);
395      }
396    }
397
398    /// <summary>
399    /// Serializes the specified object into a stream using the <see cref="XmlFormat"/>
400    /// obtained from the <see cref="ConfigurationService"/>.
401    /// </summary>
402    /// <param name="obj">The object.</param>
403    /// <param name="stream">The stream.</param>
404    /// <param name="compressionType">Type of compression, default is GZip.</param>
405    public static void Serialize(object obj, Stream stream, CompressionType compressionType = CompressionType.GZip, CancellationToken cancellationToken = default(CancellationToken)) {
406      Serialize(obj, stream, ConfigurationService.Instance.GetConfiguration(new XmlFormat()), compressionType, cancellationToken);
407    }
408
409
410    /// <summary>
411    /// Serializes the specified object into a stream.
412    /// </summary>
413    /// <param name="obj">The object.</param>
414    /// <param name="stream">The stream.</param>
415    /// <param name="config">The configuration.</param>
416    /// <param name="compressionType">Type of compression, default is GZip.</param>
417    public static void Serialize(object obj, Stream stream, Configuration config, CompressionType compressionType = CompressionType.GZip, CancellationToken cancellationToken = default(CancellationToken)) {
418      Serialize(obj, stream, config, false, compressionType, cancellationToken);
419    }
420
421    /// <summary>
422    /// Serializes the specified object into a stream.
423    /// </summary>
424    /// <param name="obj">The object.</param>
425    /// <param name="stream">The stream.</param>
426    /// <param name="config">The configuration.</param>
427    /// <param name="includeAssemblies">if set to <c>true</c> include need assemblies.</param>
428    /// <param name="compressionType">Type of compression, default is GZip.</param>
429    public static void Serialize(object obj, Stream stream, Configuration config, bool includeAssemblies, CompressionType compressionType = CompressionType.GZip, CancellationToken cancellationToken = default(CancellationToken)) {
430      try {
431        Serializer serializer = new Serializer(obj, config);
432        if (compressionType == CompressionType.Zip) {
433          Serialize(obj, stream, config, includeAssemblies, CompressionLevel.Optimal, cancellationToken);
434        } else {
435          Serialize(stream, serializer, cancellationToken);
436        }
437      } catch (PersistenceException) {
438        throw;
439      } catch (Exception e) {
440        throw new PersistenceException("Unexpected exception during Serialization.", e);
441      }
442    }
443
444    /// <summary>
445    /// Serializes the specified object into a stream.
446    /// </summary>
447    /// <param name="obj">The object.</param>
448    /// <param name="stream">The stream.</param>
449    /// <param name="config">The configuration.</param>
450    /// <param name="includeAssemblies">if set to <c>true</c> include need assemblies.</param>
451    /// <param name="types">The list of all serialized types.</param>
452    /// <param name="compressionType">Type of compression, default is GZip.</param>
453    public static void Serialize(object obj, Stream stream, Configuration config, bool includeAssemblies, out IEnumerable<Type> types,
454                                 CompressionType compressionType = CompressionType.GZip, CancellationToken cancellationToken = default(CancellationToken)) {
455      try {
456        Serializer serializer = new Serializer(obj, config);
457        if (compressionType == CompressionType.Zip) {
458          Serialize(stream, includeAssemblies, CompressionLevel.Optimal, serializer, cancellationToken);
459        } else {
460          Serialize(stream, serializer, cancellationToken);
461        }
462        types = serializer.SerializedTypes;
463      } catch (PersistenceException) {
464        throw;
465      } catch (Exception e) {
466        throw new PersistenceException("Unexpected exception during Serialization.", e);
467      }
468    }
469
470    private static void Serialize(Stream stream, Serializer serializer, CancellationToken cancellationToken = default(CancellationToken)) {
471      cancellationToken.ThrowIfCancellationRequested();
472      using (StreamWriter writer = new StreamWriter(new GZipStream(stream, CompressionMode.Compress))) {
473        serializer.InterleaveTypeInformation = true;
474        XmlGenerator generator = new XmlGenerator();
475        foreach (ISerializationToken token in serializer) {
476          cancellationToken.ThrowIfCancellationRequested();
477          string line = generator.Format(token);
478          writer.Write(line);
479        }
480        writer.Flush();
481      }
482    }
483  }
484}
Note: See TracBrowser for help on using the repository browser.