Free cookie consent management tool by TermsFeed Policy Generator

source: branches/2937_SymReg_AnalyticalQuotient/HeuristicLab.Problems.Instances.DataAnalysis/3.3/TableFileParser.cs @ 16082

Last change on this file since 16082 was 15583, checked in by swagner, 7 years ago

#2640: Updated year of copyrights in license headers

File size: 28.5 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22
23using System;
24using System.Collections;
25using System.Collections.Generic;
26using System.Diagnostics.Contracts;
27using System.Globalization;
28using System.IO;
29using System.Linq;
30using System.Text;
31
32namespace HeuristicLab.Problems.Instances.DataAnalysis {
33  public class TableFileParser : Progress<long> { // reports the number of bytes read
34    private const int BUFFER_SIZE = 65536;
35    // char used to symbolize whitespaces (no missing values can be handled with whitespaces)
36    private const char WHITESPACECHAR = (char)0;
37    private static readonly char[] POSSIBLE_SEPARATORS = new char[] { ',', ';', '\t', WHITESPACECHAR };
38    private Tokenizer tokenizer;
39    private int estimatedNumberOfLines = 200; // initial capacity for columns, will be set automatically when data is read from a file
40
41
42    private Encoding encoding = Encoding.Default;
43
44    public Encoding Encoding {
45      get { return encoding; }
46      set {
47        if (value == null) throw new ArgumentNullException("Encoding");
48        encoding = value;
49      }
50    }
51
52
53    private int rows;
54    public int Rows {
55      get { return rows; }
56      set { rows = value; }
57    }
58
59    private int columns;
60    public int Columns {
61      get { return columns; }
62      set { columns = value; }
63    }
64
65    private List<IList> values;
66    public List<IList> Values {
67      get {
68        return values;
69      }
70    }
71
72    private List<string> variableNames;
73    public IEnumerable<string> VariableNames {
74      get {
75        if (variableNames.Count > 0) return variableNames;
76        else {
77          string[] names = new string[columns];
78          for (int i = 0; i < names.Length; i++) {
79            names[i] = "X" + i.ToString("000");
80          }
81          return names;
82        }
83      }
84    }
85
86    public TableFileParser() {
87      variableNames = new List<string>();
88    }
89
90    public bool AreColumnNamesInFirstLine(string fileName) {
91      NumberFormatInfo numberFormat;
92      DateTimeFormatInfo dateTimeFormatInfo;
93      char separator;
94      DetermineFileFormat(fileName, out numberFormat, out dateTimeFormatInfo, out separator);
95      using (var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
96        return AreColumnNamesInFirstLine(stream, numberFormat, dateTimeFormatInfo, separator);
97      }
98    }
99
100    public bool AreColumnNamesInFirstLine(Stream stream) {
101      NumberFormatInfo numberFormat = NumberFormatInfo.InvariantInfo;
102      DateTimeFormatInfo dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
103      char separator = ',';
104      return AreColumnNamesInFirstLine(stream, numberFormat, dateTimeFormatInfo, separator);
105    }
106
107    public bool AreColumnNamesInFirstLine(string fileName, NumberFormatInfo numberFormat,
108                                         DateTimeFormatInfo dateTimeFormatInfo, char separator) {
109      using (var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
110        return AreColumnNamesInFirstLine(stream, numberFormat, dateTimeFormatInfo, separator);
111      }
112    }
113
114    public bool AreColumnNamesInFirstLine(Stream stream, NumberFormatInfo numberFormat,
115                                          DateTimeFormatInfo dateTimeFormatInfo, char separator) {
116      using (StreamReader reader = new StreamReader(stream, Encoding)) {
117        tokenizer = new Tokenizer(reader, numberFormat, dateTimeFormatInfo, separator);
118        return (tokenizer.PeekType() != TokenTypeEnum.Double);
119      }
120    }
121
122    /// <summary>
123    /// Parses a file and determines the format first
124    /// </summary>
125    /// <param name="fileName">file which is parsed</param>
126    /// <param name="columnNamesInFirstLine"></param>
127    public void Parse(string fileName, bool columnNamesInFirstLine, int lineLimit = -1) {
128      NumberFormatInfo numberFormat;
129      DateTimeFormatInfo dateTimeFormatInfo;
130      char separator;
131      DetermineFileFormat(fileName, out numberFormat, out dateTimeFormatInfo, out separator);
132      EstimateNumberOfLines(fileName);
133      Parse(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine, lineLimit);
134    }
135
136    /// <summary>
137    /// Parses a file with the given formats
138    /// </summary>
139    /// <param name="fileName">file which is parsed</param>
140    /// <param name="numberFormat">Format of numbers</param>
141    /// <param name="dateTimeFormatInfo">Format of datetime</param>
142    /// <param name="separator">defines the separator</param>
143    /// <param name="columnNamesInFirstLine"></param>
144    public void Parse(string fileName, NumberFormatInfo numberFormat, DateTimeFormatInfo dateTimeFormatInfo, char separator, bool columnNamesInFirstLine, int lineLimit = -1) {
145      EstimateNumberOfLines(fileName);
146      using (var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
147        Parse(stream, numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine, lineLimit);
148      }
149    }
150
151    // determines the number of newline characters in the first 64KB to guess the number of rows for a file
152    private void EstimateNumberOfLines(string fileName) {
153      var len = new System.IO.FileInfo(fileName).Length;
154      var buf = new char[1024 * 1024];
155      using (var reader = new StreamReader(fileName, Encoding)) {
156        reader.ReadBlock(buf, 0, buf.Length);
157      }
158      int numNewLine = 0;
159      int charsInCurrentLine = 0, charsInFirstLine = 0; // the first line (names) and the last line (incomplete) are not representative
160      foreach (var ch in buf) {
161        charsInCurrentLine++;
162        if (ch == '\n') {
163          if (numNewLine == 0) charsInFirstLine = charsInCurrentLine; // store the number of chars in the first line
164          charsInCurrentLine = 0;
165          numNewLine++;
166        }
167      }
168      if (numNewLine <= 1) {
169        // fail -> keep the default setting
170        return;
171      } else {
172        double charsPerLineFactor = (buf.Length - charsInFirstLine - charsInCurrentLine) / ((double)numNewLine - 1);
173        double estimatedLines = len / charsPerLineFactor;
174        estimatedNumberOfLines = (int)Math.Round(estimatedLines * 1.1); // pessimistic allocation of 110% to make sure that the list is very likely large enough
175      }
176    }
177
178    /// <summary>
179    /// Takes a Stream and parses it with default format. NumberFormatInfo.InvariantInfo, DateTimeFormatInfo.InvariantInfo and separator = ','
180    /// </summary>
181    /// <param name="stream">stream which is parsed</param>
182    /// <param name="columnNamesInFirstLine"></param>
183    public void Parse(Stream stream, bool columnNamesInFirstLine, int lineLimit = -1) {
184      NumberFormatInfo numberFormat = NumberFormatInfo.InvariantInfo;
185      DateTimeFormatInfo dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
186      char separator = ',';
187      Parse(stream, numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine, lineLimit);
188    }
189
190    /// <summary>
191    /// Parses a stream with the given formats.
192    /// </summary>
193    /// <param name="stream">Stream which is parsed</param>   
194    /// <param name="numberFormat">Format of numbers</param>
195    /// <param name="dateTimeFormatInfo">Format of datetime</param>
196    /// <param name="separator">defines the separator</param>
197    /// <param name="columnNamesInFirstLine"></param>
198    public void Parse(Stream stream, NumberFormatInfo numberFormat, DateTimeFormatInfo dateTimeFormatInfo, char separator, bool columnNamesInFirstLine, int lineLimit = -1) {
199      if (lineLimit > 0) estimatedNumberOfLines = lineLimit;
200
201      using (var reader = new StreamReader(stream)) {
202        tokenizer = new Tokenizer(reader, numberFormat, dateTimeFormatInfo, separator);
203        var strValues = new List<List<string>>();
204        values = new List<IList>();
205        Prepare(columnNamesInFirstLine, strValues);
206
207        int nLinesParsed = 0;
208        int colIdx = 0;
209        while (tokenizer.HasNext() && (lineLimit < 0 || nLinesParsed < lineLimit)) {
210          if (tokenizer.PeekType() == TokenTypeEnum.NewLine) {
211            tokenizer.Skip();
212
213            // all rows have to have the same number of values
214            // the first row defines how many elements are needed
215            if (colIdx > 0 && values.Count != colIdx) {
216              // read at least one value in the row (support for skipping empty lines)
217              Error("The first row of the dataset has " + values.Count + " columns." + Environment.NewLine +
218                    "Line " + tokenizer.CurrentLineNumber + " has " + colIdx + " columns.", "",
219                tokenizer.CurrentLineNumber);
220            }
221            OnReport(tokenizer.BytesRead);
222
223            nLinesParsed++;
224            colIdx = 0;
225          } else {
226            // read one value
227            TokenTypeEnum type;
228            string strVal;
229            double dblVal;
230            DateTime dateTimeVal;
231            tokenizer.Next(out type, out strVal, out dblVal, out dateTimeVal);
232
233            if (colIdx == values.Count) {
234              Error("The first row of the dataset has " + values.Count + " columns." + Environment.NewLine +
235                    "Line " + tokenizer.CurrentLineNumber + " has more columns.", "",
236                tokenizer.CurrentLineNumber);
237            }
238            if (!IsColumnTypeCompatible(values[colIdx], type)) {
239              values[colIdx] = strValues[colIdx];
240            }
241
242            // add the value to the column
243            AddValue(type, values[colIdx], strVal, dblVal, dateTimeVal);
244            if (!(values[colIdx] is List<string>)) { // optimization: don't store the string values in another list if the column is list<string>
245              strValues[colIdx].Add(strVal);
246            }
247            colIdx++;
248          }
249        }
250      }
251
252      if (!values.Any() || values.First().Count == 0)
253        Error("Couldn't parse data values. Probably because of incorrect number format " +
254              "(the parser expects english number format with a '.' as decimal separator).", "", tokenizer.CurrentLineNumber);
255
256      this.rows = values.First().Count;
257      this.columns = values.Count;
258
259      // replace lists with undefined type (object) with double-lists
260      for (int i = 0; i < values.Count; i++) {
261        if (values[i] is List<object>) {
262          values[i] = Enumerable.Repeat(double.NaN, rows).ToList();
263        }
264      }
265
266      // after everything has been parsed make sure the lists are as compact as possible
267      foreach (var l in values) {
268        var dblList = l as List<double>;
269        var byteList = l as List<byte>;
270        var dateList = l as List<DateTime>;
271        var stringList = l as List<string>;
272        var objList = l as List<object>;
273        if (dblList != null) dblList.TrimExcess();
274        if (byteList != null) byteList.TrimExcess();
275        if (dateList != null) dateList.TrimExcess();
276        if (stringList != null) stringList.TrimExcess();
277        if (objList != null) objList.TrimExcess();
278      }
279
280      // for large files we created a lot of memory pressure, cannot hurt to run GC.Collect here (TableFileParser is called seldomly on user interaction)
281      GC.Collect(2, GCCollectionMode.Forced);
282    }
283
284    private void Prepare(bool columnNamesInFirstLine, List<List<string>> strValues) {
285      if (columnNamesInFirstLine) {
286        ParseVariableNames();
287        if (!tokenizer.HasNext())
288          Error(
289            "Couldn't parse data values. Probably because of incorrect number format (the parser expects english number format with a '.' as decimal separator).",
290            "", tokenizer.CurrentLineNumber);
291      }
292      // read first line to determine types and allocate specific lists
293      // read values... start in first row
294      int colIdx = 0;
295      while (tokenizer.PeekType() != TokenTypeEnum.NewLine) {
296        // read one value
297        TokenTypeEnum type; string strVal; double dblVal; DateTime dateTimeVal;
298        tokenizer.Next(out type, out strVal, out dblVal, out dateTimeVal);
299
300        // initialize column
301        values.Add(CreateList(type, estimatedNumberOfLines));
302        if (type == TokenTypeEnum.String)
303          strValues.Add(new List<string>(0)); // optimization: don't store the string values in another list if the column is list<string>
304        else
305          strValues.Add(new List<string>(estimatedNumberOfLines));
306
307        AddValue(type, values[colIdx], strVal, dblVal, dateTimeVal);
308        if (type != TokenTypeEnum.String)
309          strValues[colIdx].Add(strVal);
310        colIdx++;
311      }
312      tokenizer.Skip(); // skip newline
313    }
314
315    #region type-dependent dispatch
316    private bool IsColumnTypeCompatible(IList list, TokenTypeEnum tokenType) {
317      return (list is List<object>) || // unknown lists are compatible to everything (potential conversion)
318             (list is List<string>) || // all tokens can be added to a string list
319             (tokenType == TokenTypeEnum.Missing) || // empty entries are allowed in all columns
320             (tokenType == TokenTypeEnum.Double && list is List<double>) ||
321             (tokenType == TokenTypeEnum.DateTime && list is List<DateTime>);
322    }
323
324    // all columns are converted to string columns when we find an non-empty value that has incorrect type
325    private IList ConvertToStringColumn(IList list) {
326      var dblL = list as List<double>;
327      if (dblL != null) {
328        var l = new List<string>(dblL.Capacity);
329        l.AddRange(dblL.Select(dbl => dbl.ToString()));
330        return l;
331      }
332
333      var dtL = list as List<DateTime>;
334      if (dtL != null) {
335        var l = new List<string>(dtL.Capacity);
336        l.AddRange(dtL.Select(dbl => dbl.ToString()));
337        return l;
338      }
339
340      if (list is List<string>) return list;
341
342      throw new InvalidProgramException(string.Format("Cannot convert column of type {0} to string column", list.GetType()));
343    }
344
345    private void AddValue(TokenTypeEnum type, IList list, string strVal, double dblVal, DateTime dateTimeVal) {
346      // Add value if list has a defined type
347      var dblList = list as List<double>;
348      if (dblList != null) {
349        AddValue(type, dblList, dblVal);
350        return;
351      }
352      var strList = list as List<string>;
353      if (strList != null) {
354        AddValue(type, strList, strVal);
355        return;
356      }
357      var dtList = list as List<DateTime>;
358      if (dtList != null) {
359        AddValue(type, dtList, dateTimeVal);
360        return;
361      }
362
363      // Undefined list-type
364      if (type == TokenTypeEnum.Missing) {
365        // add null to track number of missing values
366        list.Add(null);
367      } else { // first non-missing value for undefined list-type
368        var newList = ConvertList(type, list, estimatedNumberOfLines);
369        // replace list
370        var idx = values.IndexOf(list);
371        values[idx] = newList;
372        // recursively call AddValue
373        AddValue(type, newList, strVal, dblVal, dateTimeVal);
374      }
375    }
376
377    private static void AddValue(TokenTypeEnum type, List<double> list, double dblVal) {
378      Contract.Assert(type == TokenTypeEnum.Missing || type == TokenTypeEnum.Double);
379      list.Add(type == TokenTypeEnum.Missing ? double.NaN : dblVal);
380    }
381
382    private static void AddValue(TokenTypeEnum type, List<string> list, string strVal) {
383      // assumes that strVal is always set to the original token read from the input file
384      list.Add(type == TokenTypeEnum.Missing ? string.Empty : strVal);
385    }
386
387    private static void AddValue(TokenTypeEnum type, List<DateTime> list, DateTime dtVal) {
388      Contract.Assert(type == TokenTypeEnum.Missing || type == TokenTypeEnum.DateTime);
389      list.Add(type == TokenTypeEnum.Missing ? DateTime.MinValue : dtVal);
390    }
391
392    private static IList CreateList(TokenTypeEnum type, int estimatedNumberOfLines) {
393      switch (type) {
394        case TokenTypeEnum.String:
395          return new List<string>(estimatedNumberOfLines);
396        case TokenTypeEnum.Double:
397          return new List<double>(estimatedNumberOfLines);
398        case TokenTypeEnum.DateTime:
399          return new List<DateTime>(estimatedNumberOfLines);
400        case TokenTypeEnum.Missing: // List<object> represent list of unknown type
401          return new List<object>(estimatedNumberOfLines);
402        default:
403          throw new InvalidOperationException();
404      }
405    }
406
407    private static IList ConvertList(TokenTypeEnum type, IList list, int estimatedNumberOfLines) {
408      var newList = CreateList(type, estimatedNumberOfLines);
409      object missingValue = GetMissingValue(type);
410      for (int i = 0; i < list.Count; i++)
411        newList.Add(missingValue);
412      return newList;
413    }
414    private static object GetMissingValue(TokenTypeEnum type) {
415      switch (type) {
416        case TokenTypeEnum.String: return string.Empty;
417        case TokenTypeEnum.Double: return double.NaN;
418        case TokenTypeEnum.DateTime: return DateTime.MinValue;
419        default: throw new ArgumentOutOfRangeException("type", type, "No missing value defined");
420      }
421    }
422    #endregion
423
424    public static void DetermineFileFormat(string path, out NumberFormatInfo numberFormat, out DateTimeFormatInfo dateTimeFormatInfo, out char separator) {
425      DetermineFileFormat(new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), out numberFormat, out dateTimeFormatInfo, out separator);
426    }
427
428    public static void DetermineFileFormat(Stream stream, out NumberFormatInfo numberFormat, out DateTimeFormatInfo dateTimeFormatInfo, out char separator) {
429      using (StreamReader reader = new StreamReader(stream)) {
430        // skip first line
431        reader.ReadLine();
432        // read a block
433        char[] buffer = new char[BUFFER_SIZE];
434        int charsRead = reader.ReadBlock(buffer, 0, BUFFER_SIZE);
435        // count frequency of special characters
436        Dictionary<char, int> charCounts = buffer.Take(charsRead)
437          .GroupBy(c => c)
438          .ToDictionary(g => g.Key, g => g.Count());
439
440        // depending on the characters occuring in the block
441        // we distinghish a number of different cases based on the the following rules:
442        // many points => it must be English number format, the other frequently occuring char is the separator
443        // no points but many commas => this is the problematic case. Either German format (real numbers) or English format (only integer numbers) with ',' as separator
444        //   => check the line in more detail:
445        //            English: 0, 0, 0, 0
446        //            German:  0,0 0,0 0,0 ...
447        //            => if commas are followed by space => English format
448        // no points no commas => English format (only integer numbers) use the other frequently occuring char as separator
449        // in all cases only treat ' ' as separator if no other separator is possible (spaces can also occur additionally to separators)
450        if (OccurrencesOf(charCounts, '.') > 10) {
451          numberFormat = NumberFormatInfo.InvariantInfo;
452          dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
453          separator = POSSIBLE_SEPARATORS
454            .Where(c => OccurrencesOf(charCounts, c) > 10)
455            .OrderBy(c => -OccurrencesOf(charCounts, c))
456            .DefaultIfEmpty(' ')
457            .First();
458        } else if (OccurrencesOf(charCounts, ',') > 10) {
459          // no points and many commas
460          // count the number of tokens (chains of only digits and commas) that contain multiple comma characters
461          int tokensWithMultipleCommas = 0;
462          for (int i = 0; i < charsRead; i++) {
463            int nCommas = 0;
464            while (i < charsRead && (buffer[i] == ',' || Char.IsDigit(buffer[i]))) {
465              if (buffer[i] == ',') nCommas++;
466              i++;
467            }
468            if (nCommas > 2) tokensWithMultipleCommas++;
469          }
470          if (tokensWithMultipleCommas > 1) {
471            // English format (only integer values) with ',' as separator
472            numberFormat = NumberFormatInfo.InvariantInfo;
473            dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
474            separator = ',';
475          } else {
476            char[] disallowedSeparators = new char[] { ',' }; // n. def. contains a space so ' ' should be disallowed to, however existing unit tests would fail
477            // German format (real values)
478            numberFormat = NumberFormatInfo.GetInstance(new CultureInfo("de-DE"));
479            dateTimeFormatInfo = DateTimeFormatInfo.GetInstance(new CultureInfo("de-DE"));
480            separator = POSSIBLE_SEPARATORS
481              .Except(disallowedSeparators)
482              .Where(c => OccurrencesOf(charCounts, c) > 10)
483              .OrderBy(c => -OccurrencesOf(charCounts, c))
484              .DefaultIfEmpty(' ')
485              .First();
486          }
487        } else {
488          // no points and no commas => English format
489          numberFormat = NumberFormatInfo.InvariantInfo;
490          dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
491          separator = POSSIBLE_SEPARATORS
492            .Where(c => OccurrencesOf(charCounts, c) > 10)
493            .OrderBy(c => -OccurrencesOf(charCounts, c))
494            .DefaultIfEmpty(' ')
495            .First();
496        }
497      }
498    }
499
500    private static int OccurrencesOf(Dictionary<char, int> charCounts, char c) {
501      return charCounts.ContainsKey(c) ? charCounts[c] : 0;
502    }
503
504    #region tokenizer
505    // the tokenizer reads full lines and returns separated tokens in the line as well as a terminating end-of-line character
506    internal enum TokenTypeEnum {
507      NewLine, String, Double, DateTime, Missing
508    }
509
510    internal class Tokenizer {
511      private StreamReader reader;
512      // we assume that a buffer of 1024 tokens for a line is sufficient most of the time (the buffer is increased below if necessary)
513      private TokenTypeEnum[] tokenTypes = new TokenTypeEnum[1024];
514      private string[] stringVals = new string[1024];
515      private double[] doubleVals = new double[1024];
516      private DateTime[] dateTimeVals = new DateTime[1024];
517      private int tokenPos;
518      private int numTokens;
519      private NumberFormatInfo numberFormatInfo;
520      private DateTimeFormatInfo dateTimeFormatInfo;
521      private char separator;
522
523      // arrays for string.Split()
524      private readonly char[] whiteSpaceSeparators = new char[0]; // string split uses separators as default
525      private readonly char[] separators;
526
527      private int currentLineNumber = 0;
528      public int CurrentLineNumber {
529        get { return currentLineNumber; }
530        private set { currentLineNumber = value; }
531      }
532      private string currentLine;
533      public string CurrentLine {
534        get { return currentLine; }
535        private set { currentLine = value; }
536      }
537      public long BytesRead {
538        get;
539        private set;
540      }
541
542      public Tokenizer(StreamReader reader, NumberFormatInfo numberFormatInfo, DateTimeFormatInfo dateTimeFormatInfo, char separator) {
543        this.reader = reader;
544        this.numberFormatInfo = numberFormatInfo;
545        this.dateTimeFormatInfo = dateTimeFormatInfo;
546        this.separator = separator;
547        this.separators = new char[] { separator };
548        ReadNextTokens();
549      }
550
551      public bool HasNext() {
552        return numTokens > tokenPos || !reader.EndOfStream;
553      }
554
555      public TokenTypeEnum PeekType() {
556        return tokenTypes[tokenPos];
557      }
558
559      public void Skip() {
560        // simply skips one token without returning the result values
561        tokenPos++;
562        if (numTokens == tokenPos) {
563          ReadNextTokens();
564        }
565      }
566
567      public void Next(out TokenTypeEnum type, out string strVal, out double dblVal, out DateTime dateTimeVal) {
568        type = tokenTypes[tokenPos];
569        strVal = stringVals[tokenPos];
570        dblVal = doubleVals[tokenPos];
571        dateTimeVal = dateTimeVals[tokenPos];
572        Skip();
573      }
574
575      private void ReadNextTokens() {
576        if (!reader.EndOfStream) {
577          CurrentLine = reader.ReadLine();
578          CurrentLineNumber++;
579          if (reader.BaseStream.CanSeek) {
580            BytesRead = reader.BaseStream.Position;
581          } else {
582            BytesRead += CurrentLine.Length + 2; // guess
583          }
584          int i = 0;
585          if (!string.IsNullOrWhiteSpace(CurrentLine)) {
586            foreach (var tok in Split(CurrentLine)) {
587              TokenTypeEnum type;
588              double doubleVal;
589              DateTime dateTimeValue;
590              type = TokenTypeEnum.String; // default
591              stringVals[i] = tok.Trim();
592              if (double.TryParse(tok, NumberStyles.Float, numberFormatInfo, out doubleVal)) {
593                type = TokenTypeEnum.Double;
594                doubleVals[i] = doubleVal;
595              } else if (DateTime.TryParse(tok, dateTimeFormatInfo, DateTimeStyles.NoCurrentDateDefault, out dateTimeValue)
596                && (dateTimeValue.Year > 1 || dateTimeValue.Month > 1 || dateTimeValue.Day > 1)// if no date is given it is returned as 1.1.0001 -> don't allow this
597                ) {
598                type = TokenTypeEnum.DateTime;
599                dateTimeVals[i] = dateTimeValue;
600              } else if (string.IsNullOrWhiteSpace(tok)) {
601                type = TokenTypeEnum.Missing;
602              }
603
604              // couldn't parse the token as an int or float number or datetime value so return a string token
605
606              tokenTypes[i] = type;
607              i++;
608
609              if (i >= tokenTypes.Length) {
610                // increase buffer size if necessary
611                IncreaseCapacity(ref tokenTypes);
612                IncreaseCapacity(ref doubleVals);
613                IncreaseCapacity(ref stringVals);
614                IncreaseCapacity(ref dateTimeVals);
615              }
616            }
617          }
618          tokenTypes[i] = TokenTypeEnum.NewLine;
619          numTokens = i + 1;
620          tokenPos = 0;
621        }
622      }
623
624      private IEnumerable<string> Split(string line) {
625        return separator == WHITESPACECHAR ?
626          line.Split(whiteSpaceSeparators, StringSplitOptions.RemoveEmptyEntries) :
627          line.Split(separators);
628      }
629
630      private static void IncreaseCapacity<T>(ref T[] arr) {
631        int n = (int)Math.Floor(arr.Length * 1.7); // guess
632        T[] arr2 = new T[n];
633        Array.Copy(arr, arr2, arr.Length);
634        arr = arr2;
635      }
636    }
637    #endregion
638
639    #region parsing
640
641    private void ParseVariableNames() {
642      // the first line must contain variable names
643      List<string> varNames = new List<string>();
644
645      TokenTypeEnum type;
646      string strVal;
647      double dblVal;
648      DateTime dateTimeVal;
649
650      tokenizer.Next(out type, out strVal, out dblVal, out dateTimeVal);
651
652      // the first token must be a variable name
653      if (type != TokenTypeEnum.String)
654        throw new ArgumentException("Error: Expected " + TokenTypeEnum.String + " got " + type);
655      varNames.Add(strVal);
656
657      while (tokenizer.HasNext() && tokenizer.PeekType() != TokenTypeEnum.NewLine) {
658        tokenizer.Next(out type, out strVal, out dblVal, out dateTimeVal);
659        varNames.Add(strVal);
660      }
661      ExpectType(TokenTypeEnum.NewLine);
662
663      variableNames = varNames;
664    }
665
666    private void ExpectType(TokenTypeEnum expectedToken) {
667      if (tokenizer.PeekType() != expectedToken)
668        throw new ArgumentException("Error: Expected " + expectedToken + " got " + tokenizer.PeekType());
669      tokenizer.Skip();
670    }
671
672    private void Error(string message, string token, int lineNumber) {
673      throw new IOException(string.Format("Error while parsing. {0} (token: {1} lineNumber: {2}).", message, token, lineNumber));
674    }
675    #endregion
676  }
677}
Note: See TracBrowser for help on using the repository browser.