Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/sources/HeuristicLab.Persistence/3.3/Default/CompositeSerializers/CompactNumberArray2StringSerializer.cs @ 3946

Last change on this file since 3946 was 3946, checked in by epitzer, 14 years ago

Revise CompactNumberArray2StringSerializer: split text into several smaller strings to mitigate memory problems for very large arrays (#1138)

File size: 5.8 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2010 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Linq;
24using HeuristicLab.Persistence.Interfaces;
25using HeuristicLab.Persistence.Core;
26using System.Collections.Generic;
27using System.Reflection;
28using System.Globalization;
29using System.Text;
30using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
31using HeuristicLab.Persistence.Auxiliary;
32
33namespace HeuristicLab.Persistence.Default.CompositeSerializers {
34
35  [StorableClass]
36  public sealed class CompactNumberArray2StringSerializer : ICompositeSerializer {
37
38    public const int SPLIT_THRESHOLD = 1024 * 1024;
39
40    public int Priority {
41      get { return 200; }
42    }
43
44    private static readonly Number2StringSerializer numberConverter =
45      new Number2StringSerializer();
46
47    public bool CanSerialize(Type type) {
48      return
49        (type.IsArray || type == typeof(Array)) &&
50        numberConverter.CanSerialize(type.GetElementType());
51    }
52
53    public string JustifyRejection(Type type) {
54      if (!type.IsArray && type != typeof(Array))
55        return "not an array";
56      return string.Format("number converter cannot serialize elements: " +
57        numberConverter.JustifyRejection(type.GetElementType()));
58    }
59
60    public IEnumerable<Tag> CreateMetaInfo(object obj) {
61      Array a = (Array)obj;
62      int[] lengths = new int[a.Rank];
63      int[] lowerBounds = new int[a.Rank];
64      StringBuilder sb = new StringBuilder(a.Rank * 6);
65      sb.Append(a.Rank).Append(';');
66      for (int i = 0; i < a.Rank; i++) {
67        sb.Append(a.GetLength(i)).Append(';');
68        lengths[i] = a.GetLength(i);
69      }
70      for (int i = 0; i < a.Rank; i++) {
71        sb.Append(a.GetLowerBound(i)).Append(';');
72        lowerBounds[i] = a.GetLowerBound(i);
73      }
74      yield return new Tag(sb.ToString());
75      int nElements = 1;
76      for (int i = 0; i < a.Rank; i++) {
77        lowerBounds[i] = a.GetLowerBound(i);
78        lengths[i] = a.GetLength(i);
79        nElements *= lengths[i];
80      }
81      sb = new StringBuilder(Math.Min(nElements * 3, SPLIT_THRESHOLD));
82      int[] positions = (int[])lowerBounds.Clone();
83      while (positions[a.Rank - 1] < lengths[a.Rank - 1] + lowerBounds[a.Rank - 1]) {
84        sb.Append(numberConverter.Format(a.GetValue(positions))).Append(';');
85        if (sb.Length > SPLIT_THRESHOLD && sb.Length > sb.Capacity - 18) {
86          yield return new Tag(sb.ToString());
87          sb = new StringBuilder(Math.Min(nElements * 3, SPLIT_THRESHOLD));
88        }
89        positions[0] += 1;
90        for (int i = 0; i < a.Rank - 1; i++) {
91          if (positions[i] >= lengths[i] + lowerBounds[i]) {
92            positions[i] = lowerBounds[i];
93            positions[i + 1] += 1;
94          } else {
95            break;
96          }
97        }
98      }
99      if (sb.Length > 0)
100        yield return new Tag(sb.ToString());
101    }
102
103    private static Tag[] emptyTag = new Tag[0];
104    public IEnumerable<Tag> Decompose(object obj) {
105      return emptyTag;
106    }
107
108    public object CreateInstance(Type type, IEnumerable<Tag> metaInfo) {
109      try {
110        var tagIter = metaInfo.GetEnumerator();
111        tagIter.MoveNext();
112        var valueIter = ((string)tagIter.Current.Value).GetSplitEnumerator(';');
113        valueIter.MoveNext();
114        int rank = int.Parse(valueIter.Current);
115        int[] lengths = new int[rank];
116        int[] lowerBounds = new int[rank];
117        for (int i = 0; i < rank; i++) {
118          valueIter.MoveNext();
119          lengths[i] = int.Parse(valueIter.Current);
120        }
121        for (int i = 0; i < rank; i++) {
122          valueIter.MoveNext();
123          lowerBounds[i] = int.Parse(valueIter.Current);
124        }
125        Type elementType = type.GetElementType();
126        Array a = Array.CreateInstance(elementType, lengths, lowerBounds);
127        if (a == null) throw new PersistenceException("invalid instance data type, expected array");
128        int[] positions = (int[])lowerBounds.Clone();
129        while (tagIter.MoveNext()) {
130          valueIter = ((string)tagIter.Current.Value).GetSplitEnumerator(';');
131          while (valueIter.MoveNext()) {
132            a.SetValue(numberConverter.Parse(valueIter.Current, elementType), positions);
133            positions[0] += 1;
134            for (int i = 0; i < a.Rank - 1; i++) {
135              if (positions[i] >= lengths[i] + lowerBounds[i]) {
136                positions[i + 1] += 1;
137                positions[i] = lowerBounds[i];
138              } else {
139                break;
140              }
141            }
142          }
143        }
144        return a;
145      } catch (InvalidOperationException e) {
146        throw new PersistenceException("Insuffictient data to deserialize compact array", e);
147      } catch (InvalidCastException e) {
148        throw new PersistenceException("Invalid element data during compact array deserialization", e);
149      }
150    }
151
152    public void Populate(object instance, IEnumerable<Tag> tags, Type type) {
153      // Nothing to do. Arrays are populated during instance creation;
154    }
155
156  }
157
158}
Note: See TracBrowser for help on using the repository browser.