Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/sources/HeuristicLab.Persistence/3.3/Default/CompositeSerializers/CompactNumberArray2StringSerializer.cs @ 4047

Last change on this file since 4047 was 3946, checked in by epitzer, 15 years ago

Revise CompactNumberArray2StringSerializer: split text into several smaller strings to mitigate memory problems for very large arrays (#1138)

File size: 5.8 KB
RevLine 
[3742]1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2010 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
[3937]23using System.Linq;
[1567]24using HeuristicLab.Persistence.Interfaces;
25using HeuristicLab.Persistence.Core;
26using System.Collections.Generic;
27using System.Reflection;
28using System.Globalization;
29using System.Text;
[1823]30using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
[3937]31using HeuristicLab.Persistence.Auxiliary;
[1567]32
[1823]33namespace HeuristicLab.Persistence.Default.CompositeSerializers {
[1567]34
[3017]35  [StorableClass]
[3946]36  public sealed class CompactNumberArray2StringSerializer : ICompositeSerializer {
[1567]37
[3946]38    public const int SPLIT_THRESHOLD = 1024 * 1024;
39
[1567]40    public int Priority {
41      get { return 200; }
42    }
43
[1823]44    private static readonly Number2StringSerializer numberConverter =
45      new Number2StringSerializer();
[1567]46
[1823]47    public bool CanSerialize(Type type) {
[1567]48      return
49        (type.IsArray || type == typeof(Array)) &&
[1823]50        numberConverter.CanSerialize(type.GetElementType());
[1567]51    }
52
[2993]53    public string JustifyRejection(Type type) {
54      if (!type.IsArray && type != typeof(Array))
55        return "not an array";
56      return string.Format("number converter cannot serialize elements: " +
57        numberConverter.JustifyRejection(type.GetElementType()));
58    }
59
[1567]60    public IEnumerable<Tag> CreateMetaInfo(object obj) {
61      Array a = (Array)obj;
62      int[] lengths = new int[a.Rank];
63      int[] lowerBounds = new int[a.Rank];
[3946]64      StringBuilder sb = new StringBuilder(a.Rank * 6);
[1567]65      sb.Append(a.Rank).Append(';');
66      for (int i = 0; i < a.Rank; i++) {
67        sb.Append(a.GetLength(i)).Append(';');
68        lengths[i] = a.GetLength(i);
69      }
70      for (int i = 0; i < a.Rank; i++) {
71        sb.Append(a.GetLowerBound(i)).Append(';');
72        lowerBounds[i] = a.GetLowerBound(i);
73      }
[3946]74      yield return new Tag(sb.ToString());
75      int nElements = 1;
76      for (int i = 0; i < a.Rank; i++) {
77        lowerBounds[i] = a.GetLowerBound(i);
78        lengths[i] = a.GetLength(i);
79        nElements *= lengths[i];
80      }
81      sb = new StringBuilder(Math.Min(nElements * 3, SPLIT_THRESHOLD));
[1567]82      int[] positions = (int[])lowerBounds.Clone();
83      while (positions[a.Rank - 1] < lengths[a.Rank - 1] + lowerBounds[a.Rank - 1]) {
84        sb.Append(numberConverter.Format(a.GetValue(positions))).Append(';');
[3946]85        if (sb.Length > SPLIT_THRESHOLD && sb.Length > sb.Capacity - 18) {
86          yield return new Tag(sb.ToString());
87          sb = new StringBuilder(Math.Min(nElements * 3, SPLIT_THRESHOLD));
88        }
[1567]89        positions[0] += 1;
90        for (int i = 0; i < a.Rank - 1; i++) {
91          if (positions[i] >= lengths[i] + lowerBounds[i]) {
92            positions[i] = lowerBounds[i];
93            positions[i + 1] += 1;
94          } else {
95            break;
96          }
97        }
98      }
[3946]99      if (sb.Length > 0)
100        yield return new Tag(sb.ToString());
[1567]101    }
102
[3946]103    private static Tag[] emptyTag = new Tag[0];
[1567]104    public IEnumerable<Tag> Decompose(object obj) {
[3946]105      return emptyTag;
[1567]106    }
107
108    public object CreateInstance(Type type, IEnumerable<Tag> metaInfo) {
[1625]109      try {
110        var tagIter = metaInfo.GetEnumerator();
[1703]111        tagIter.MoveNext();
[3937]112        var valueIter = ((string)tagIter.Current.Value).GetSplitEnumerator(';');
[1567]113        valueIter.MoveNext();
[3946]114        int rank = int.Parse(valueIter.Current);
[1625]115        int[] lengths = new int[rank];
116        int[] lowerBounds = new int[rank];
117        for (int i = 0; i < rank; i++) {
118          valueIter.MoveNext();
[3946]119          lengths[i] = int.Parse(valueIter.Current);
[1625]120        }
121        for (int i = 0; i < rank; i++) {
122          valueIter.MoveNext();
[3946]123          lowerBounds[i] = int.Parse(valueIter.Current);
[1625]124        }
125        Type elementType = type.GetElementType();
126        Array a = Array.CreateInstance(elementType, lengths, lowerBounds);
[3946]127        if (a == null) throw new PersistenceException("invalid instance data type, expected array");
[1625]128        int[] positions = (int[])lowerBounds.Clone();
[3946]129        while (tagIter.MoveNext()) {
130          valueIter = ((string)tagIter.Current.Value).GetSplitEnumerator(';');
131          while (valueIter.MoveNext()) {
132            a.SetValue(numberConverter.Parse(valueIter.Current, elementType), positions);
133            positions[0] += 1;
134            for (int i = 0; i < a.Rank - 1; i++) {
135              if (positions[i] >= lengths[i] + lowerBounds[i]) {
136                positions[i + 1] += 1;
137                positions[i] = lowerBounds[i];
138              } else {
139                break;
140              }
[1625]141            }
[1567]142          }
143        }
[1625]144        return a;
145      } catch (InvalidOperationException e) {
[3946]146        throw new PersistenceException("Insuffictient data to deserialize compact array", e);
[1625]147      } catch (InvalidCastException e) {
148        throw new PersistenceException("Invalid element data during compact array deserialization", e);
[1703]149      }
[1567]150    }
151
152    public void Populate(object instance, IEnumerable<Tag> tags, Type type) {
[3946]153      // Nothing to do. Arrays are populated during instance creation;
[1567]154    }
155
156  }
157
158}
Note: See TracBrowser for help on using the repository browser.