Changeset 13974 for stable/HeuristicLab.Problems.Instances.DataAnalysis/3.3/TableFileParser.cs

Timestamp:

07/02/16 08:15:07 (8 years ago)

Author:

gkronber

Message:

#2071: merged r13411,r13413,r13414,r13415,r13419,r13440,r13441,r13442,r13445,r13447,r13525,r13526,r13529,r13584,r13901,r13925 from trunk to stable

Location:

stable

Files:

• . (modified) (1 prop)
• HeuristicLab.Problems.Instances.DataAnalysis (modified) (1 prop)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/TableFileParser.cs (modified) (14 diffs)

stable/HeuristicLab.Problems.Instances.DataAnalysis/3.3/TableFileParser.cs

@@ -24 +24 @@
 using System.Collections;
 using System.Collections.Generic;
+using System.Diagnostics.Contracts;
 using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Runtime.Serialization;
+using System.Text;
 
 namespace HeuristicLab.Problems.Instances.DataAnalysis {
-  public class TableFileParser {
+  public class TableFileParser : Progress<long> { // reports the number of bytes read
     private const int BUFFER_SIZE = 65536;
     // char used to symbolize whitespaces (no missing values can be handled with whitespaces)
@@ -36 +38 @@
     private static readonly char[] POSSIBLE_SEPARATORS = new char[] { ',', ';', '\t', WHITESPACECHAR };
     private Tokenizer tokenizer;
-    private List<List<object>> rowValues;
+    private int estimatedNumberOfLines = 200; // initial capacity for columns, will be set automatically when data is read from a file
+
+
+    private Encoding encoding = Encoding.Default;
+
+    public Encoding Encoding {
+      get { return encoding; }
+      set {
+        if (value == null) throw new ArgumentNullException("Encoding");
+        encoding = value;
+      }
+    }
+
 
     private int rows;
@@ -72 +86 @@
 
     public TableFileParser() {
-      rowValues = new List<List<object>>();
       variableNames = new List<string>();
     }
@@ -102 +115 @@
     public bool AreColumnNamesInFirstLine(Stream stream, NumberFormatInfo numberFormat,
                                           DateTimeFormatInfo dateTimeFormatInfo, char separator) {
-      using (StreamReader reader = new StreamReader(stream)) {
+      using (StreamReader reader = new StreamReader(stream, Encoding)) {
         tokenizer = new Tokenizer(reader, numberFormat, dateTimeFormatInfo, separator);
-        return tokenizer.Peek().type != TokenTypeEnum.Double;
+        return (tokenizer.PeekType() != TokenTypeEnum.Double);
       }
     }
@@ -113 +126 @@
     /// <param name="fileName">file which is parsed</param>
     /// <param name="columnNamesInFirstLine"></param>
-    public void Parse(string fileName, bool columnNamesInFirstLine) {
+    public void Parse(string fileName, bool columnNamesInFirstLine, int lineLimit = -1) {
       NumberFormatInfo numberFormat;
       DateTimeFormatInfo dateTimeFormatInfo;
       char separator;
       DetermineFileFormat(fileName, out numberFormat, out dateTimeFormatInfo, out separator);
-      Parse(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine);
+      EstimateNumberOfLines(fileName);
+      Parse(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine, lineLimit);
     }
 
@@ -129 +143 @@
     /// <param name="separator">defines the separator</param>
     /// <param name="columnNamesInFirstLine"></param>
-    public void Parse(string fileName, NumberFormatInfo numberFormat, DateTimeFormatInfo dateTimeFormatInfo, char separator, bool columnNamesInFirstLine) {
+    public void Parse(string fileName, NumberFormatInfo numberFormat, DateTimeFormatInfo dateTimeFormatInfo, char separator, bool columnNamesInFirstLine, int lineLimit = -1) {
+      EstimateNumberOfLines(fileName);
       using (var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
-        Parse(stream, numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine);
+        Parse(stream, numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine, lineLimit);
+      }
+    }
+
+    // determines the number of newline characters in the first 64KB to guess the number of rows for a file
+    private void EstimateNumberOfLines(string fileName) {
+      var len = new System.IO.FileInfo(fileName).Length;
+      var buf = new char[1024 * 1024];
+      using (var reader = new StreamReader(fileName, Encoding)) {
+        reader.ReadBlock(buf, 0, buf.Length);
+      }
+      int numNewLine = 0;
+      int charsInCurrentLine = 0, charsInFirstLine = 0; // the first line (names) and the last line (incomplete) are not representative
+      foreach (var ch in buf) {
+        charsInCurrentLine++;
+        if (ch == '\n') {
+          if (numNewLine == 0) charsInFirstLine = charsInCurrentLine; // store the number of chars in the first line
+          charsInCurrentLine = 0;
+          numNewLine++;
+        }
+      }
+      if (numNewLine <= 1) {
+        // fail -> keep the default setting
+        return;
+      } else {
+        double charsPerLineFactor = (buf.Length - charsInFirstLine - charsInCurrentLine) / ((double)numNewLine - 1);
+        double estimatedLines = len / charsPerLineFactor;
+        estimatedNumberOfLines = (int)Math.Round(estimatedLines * 1.1); // pessimistic allocation of 110% to make sure that the list is very likely large enough
       }
     }
@@ -140 +182 @@
     /// <param name="stream">stream which is parsed</param>
     /// <param name="columnNamesInFirstLine"></param>
-    public void Parse(Stream stream, bool columnNamesInFirstLine) {
+    public void Parse(Stream stream, bool columnNamesInFirstLine, int lineLimit = -1) {
       NumberFormatInfo numberFormat = NumberFormatInfo.InvariantInfo;
       DateTimeFormatInfo dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
       char separator = ',';
-      Parse(stream, numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine);
+      Parse(stream, numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine, lineLimit);
     }
 
@@ -155 +197 @@
     /// <param name="separator">defines the separator</param>
     /// <param name="columnNamesInFirstLine"></param>
-    public void Parse(Stream stream, NumberFormatInfo numberFormat, DateTimeFormatInfo dateTimeFormatInfo, char separator, bool columnNamesInFirstLine) {
-      using (StreamReader reader = new StreamReader(stream)) {
+    public void Parse(Stream stream, NumberFormatInfo numberFormat, DateTimeFormatInfo dateTimeFormatInfo, char separator, bool columnNamesInFirstLine, int lineLimit = -1) {
+      using (StreamReader reader = new StreamReader(stream, Encoding)) {
         tokenizer = new Tokenizer(reader, numberFormat, dateTimeFormatInfo, separator);
-        // parse the file
-        Parse(columnNamesInFirstLine);
-      }
-
-      // translate the list of samples into a DoubleMatrixData item
-      rows = rowValues.Count;
-      columns = rowValues[0].Count;
-      values = new List<IList>();
-
-      //create columns
-      for (int col = 0; col < columns; col++) {
-        var types = rowValues.Select(r => r[col]).Where(v => v != null && v as string != string.Empty).Take(100).Select(v => v.GetType());
-        if (!types.Any()) {
-          values.Add(new List<string>());
-          continue;
+        values = new List<IList>();
+        if (lineLimit > 0) estimatedNumberOfLines = lineLimit;
+
+        if (columnNamesInFirstLine) {
+          ParseVariableNames();
+          if (!tokenizer.HasNext())
+            Error(
+              "Couldn't parse data values. Probably because of incorrect number format (the parser expects english number format with a '.' as decimal separator).",
+              "", tokenizer.CurrentLineNumber);
         }
 
-        var columnType = types.GroupBy(v => v).OrderBy(v => v.Count()).Last().Key;
-        if (columnType == typeof(double)) values.Add(new List<double>());
-        else if (columnType == typeof(DateTime)) values.Add(new List<DateTime>());
-        else if (columnType == typeof(string)) values.Add(new List<string>());
-        else throw new InvalidOperationException();
-      }
-
-
-
-      //fill with values
-      foreach (List<object> row in rowValues) {
-        int columnIndex = 0;
-        foreach (object element in row) {
-          if (values[columnIndex] is List<double> && !(element is double))
-            values[columnIndex].Add(double.NaN);
-          else if (values[columnIndex] is List<DateTime> && !(element is DateTime))
-            values[columnIndex].Add(DateTime.MinValue);
-          else if (values[columnIndex] is List<string> && !(element is string))
-            values[columnIndex].Add(element.ToString());
-          else
-            values[columnIndex].Add(element);
-          columnIndex++;
+
+        // read values... start in first row 
+        int nLinesParsed = 0;
+        int colIdx = 0;
+        int numValuesInFirstRow = columnNamesInFirstLine ? variableNames.Count : -1; // number of variables or inizialize based on first row of values (-1)
+        while (tokenizer.HasNext() && (lineLimit < 0 || nLinesParsed < lineLimit)) {
+          if (tokenizer.PeekType() == TokenTypeEnum.NewLine) {
+            tokenizer.Skip();
+
+            // all rows have to have the same number of values
+            // the first row defines how many samples are needed
+            if (numValuesInFirstRow < 0) numValuesInFirstRow = values.Count; // set to number of colums in the first row
+            else if (colIdx > 0 && numValuesInFirstRow != colIdx) { // read at least one value in the row (support for skipping empty lines)
+              Error("The first row of the dataset has " + numValuesInFirstRow + " columns." + Environment.NewLine +
+                    "Line " + tokenizer.CurrentLineNumber + " has " + colIdx + " columns.", "",
+                    tokenizer.CurrentLineNumber);
+            }
+            OnReport(tokenizer.BytesRead);
+
+            nLinesParsed++;
+            colIdx = 0;
+          } else {
+            // read one value
+            TokenTypeEnum type; string strVal; double dblVal; DateTime dateTimeVal;
+            tokenizer.Next(out type, out strVal, out dblVal, out dateTimeVal);
+
+            // initialize columns on the first row (fixing data types as presented in the first row...)
+            if (nLinesParsed == 0) {
+              values.Add(CreateList(type, estimatedNumberOfLines));
+            } else if (colIdx == values.Count) {
+              Error("The first row of the dataset has " + numValuesInFirstRow + " columns." + Environment.NewLine +
+                    "Line " + tokenizer.CurrentLineNumber + " has more columns.", "",
+                tokenizer.CurrentLineNumber);
+            }
+            if (!IsColumnTypeCompatible(values[colIdx], type)) {
+              values[colIdx] = ConvertToStringColumn(values[colIdx]);
+            }
+            // add the value to the column
+            AddValue(type, values[colIdx++], strVal, dblVal, dateTimeVal);
+          }
         }
-      }
-    }
+
+        if (!values.Any() || values.First().Count == 0)
+          Error("Couldn't parse data values. Probably because of incorrect number format " +
+                "(the parser expects english number format with a '.' as decimal separator).", "", tokenizer.CurrentLineNumber);
+      }
+
+      this.rows = values.First().Count;
+      this.columns = values.Count;
+
+      // after everything has been parsed make sure the lists are as compact as possible
+      foreach (var l in values) {
+        var dblList = l as List<double>;
+        var byteList = l as List<byte>;
+        var dateList = l as List<DateTime>;
+        var stringList = l as List<string>;
+        var objList = l as List<object>;
+        if (dblList != null) dblList.TrimExcess();
+        if (byteList != null) byteList.TrimExcess();
+        if (dateList != null) dateList.TrimExcess();
+        if (stringList != null) stringList.TrimExcess();
+        if (objList != null) objList.TrimExcess();
+      }
+
+      // for large files we created a lot of memory pressure, cannot hurt to run GC.Collect here (TableFileParser is called seldomly on user interaction)
+      GC.Collect(2, GCCollectionMode.Forced);
+    }
+
+    #region type-dependent dispatch
+    private bool IsColumnTypeCompatible(IList list, TokenTypeEnum tokenType) {
+      return (list is List<string>) || // all tokens can be added to a string list
+             (tokenType == TokenTypeEnum.Missing) || // empty entries are allowed in all columns
+             (tokenType == TokenTypeEnum.Double && list is List<double>) ||
+             (tokenType == TokenTypeEnum.DateTime && list is List<DateTime>);
+    }
+
+    // all columns are converted to string columns when we find an non-empty value that has incorrect type
+    private IList ConvertToStringColumn(IList list) {
+      var dblL = list as List<double>;
+      if (dblL != null) {
+        var l = new List<string>(dblL.Capacity);
+        l.AddRange(dblL.Select(dbl => dbl.ToString()));
+        return l;
+      }
+
+      var dtL = list as List<DateTime>;
+      if (dtL != null) {
+        var l = new List<string>(dtL.Capacity);
+        l.AddRange(dtL.Select(dbl => dbl.ToString()));
+        return l;
+      }
+
+      if (list is List<string>) return list;
+
+      throw new InvalidProgramException(string.Format("Cannot convert column of type {0} to string column", list.GetType()));
+    }
+
+    private void AddValue(TokenTypeEnum type, IList list, string strVal, double dblVal, DateTime dateTimeVal) {
+      var dblList = list as List<double>;
+      if (dblList != null) {
+        AddValue(type, dblList, dblVal);
+        return;
+      }
+
+      var strList = list as List<string>;
+      if (strList != null) {
+        AddValue(type, strList, strVal);
+        return;
+      }
+      var dtList = list as List<DateTime>;
+      if (dtList != null) {
+        AddValue(type, dtList, dateTimeVal);
+        return;
+      }
+
+      list.Add(strVal); // assumes List<object>
+    }
+
+    private void AddValue(TokenTypeEnum type, List<double> list, double dblVal) {
+      Contract.Assert(type == TokenTypeEnum.Missing || type == TokenTypeEnum.Double);
+      list.Add(type == TokenTypeEnum.Missing ? double.NaN : dblVal);
+    }
+
+    private void AddValue(TokenTypeEnum type, List<string> list, string strVal) {
+      // assumes that strVal is always set to the original token read from the input file
+      list.Add(type == TokenTypeEnum.Missing ? string.Empty : strVal);
+    }
+
+    private void AddValue(TokenTypeEnum type, List<DateTime> list, DateTime dtVal) {
+      Contract.Assert(type == TokenTypeEnum.Missing || type == TokenTypeEnum.DateTime);
+      list.Add(type == TokenTypeEnum.Missing ? DateTime.MinValue : dtVal);
+    }
+
+    private IList CreateList(TokenTypeEnum type, int estimatedNumberOfLines) {
+      switch (type) {
+        case TokenTypeEnum.String:
+          return new List<string>(estimatedNumberOfLines);
+        case TokenTypeEnum.Double:
+        case TokenTypeEnum.Missing: // assume double columns
+          return new List<double>(estimatedNumberOfLines);
+        case TokenTypeEnum.DateTime:
+          return new List<DateTime>(estimatedNumberOfLines);
+        default:
+          throw new InvalidOperationException();
+      }
+    }
+    #endregion
 
     public static void DetermineFileFormat(string path, out NumberFormatInfo numberFormat, out DateTimeFormatInfo dateTimeFormatInfo, out char separator) {
@@ -253 +411 @@
             separator = ',';
           } else {
-            char[] disallowedSeparators = new char[] { ',' };
+            char[] disallowedSeparators = new char[] { ',' }; // n. def. contains a space so ' ' should be disallowed to, however existing unit tests would fail
             // German format (real values)
             numberFormat = NumberFormatInfo.GetInstance(new CultureInfo("de-DE"));
@@ -282 +440 @@
 
     #region tokenizer
+    // the tokenizer reads full lines and returns separated tokens in the line as well as a terminating end-of-line character
     internal enum TokenTypeEnum {
-      NewLine, Separator, String, Double, DateTime
-    }
-
-    internal class Token {
-      public TokenTypeEnum type;
-      public string stringValue;
-      public double doubleValue;
-      public DateTime dateTimeValue;
-
-      public Token(TokenTypeEnum type, string value) {
-        this.type = type;
-        stringValue = value;
-        dateTimeValue = DateTime.MinValue;
-        doubleValue = 0.0;
-      }
-
-      public override string ToString() {
-        return stringValue;
-      }
-    }
-
+      NewLine, String, Double, DateTime, Missing
+    }
 
     internal class Tokenizer {
       private StreamReader reader;
-      private List<Token> tokens;
+      // we assume that a buffer of 1024 tokens for a line is sufficient most of the time (the buffer is increased below if necessary)
+      private TokenTypeEnum[] tokenTypes = new TokenTypeEnum[1024];
+      private string[] stringVals = new string[1024];
+      private double[] doubleVals = new double[1024];
+      private DateTime[] dateTimeVals = new DateTime[1024];
+      private int tokenPos;
+      private int numTokens;
       private NumberFormatInfo numberFormatInfo;
       private DateTimeFormatInfo dateTimeFormatInfo;
       private char separator;
-      private const string INTERNAL_SEPARATOR = "#";
+
+      // arrays for string.Split()
+      private readonly char[] whiteSpaceSeparators = new char[0]; // string split uses separators as default
+      private readonly char[] separators;
 
       private int currentLineNumber = 0;
@@ -323 +472 @@
         private set { currentLine = value; }
       }
-
-      private Token newlineToken;
-      public Token NewlineToken {
-        get { return newlineToken; }
-        private set { newlineToken = value; }
-      }
-      private Token separatorToken;
-      public Token SeparatorToken {
-        get { return separatorToken; }
-        private set { separatorToken = value; }
+      public long BytesRead {
+        get;
+        private set;
       }
 
@@ -340 +482 @@
         this.dateTimeFormatInfo = dateTimeFormatInfo;
         this.separator = separator;
-        separatorToken = new Token(TokenTypeEnum.Separator, INTERNAL_SEPARATOR);
-        newlineToken = new Token(TokenTypeEnum.NewLine, Environment.NewLine);
-        tokens = new List<Token>();
+        this.separators = new char[] { separator };
         ReadNextTokens();
+      }
+
+      public bool HasNext() {
+        return numTokens > tokenPos || !reader.EndOfStream;
+      }
+
+      public TokenTypeEnum PeekType() {
+        return tokenTypes[tokenPos];
+      }
+
+      public void Skip() {
+        // simply skips one token without returning the result values
+        tokenPos++;
+        if (numTokens == tokenPos) {
+          ReadNextTokens();
+        }
+      }
+
+      public void Next(out TokenTypeEnum type, out string strVal, out double dblVal, out DateTime dateTimeVal) {
+        type = tokenTypes[tokenPos];
+        strVal = stringVals[tokenPos];
+        dblVal = doubleVals[tokenPos];
+        dateTimeVal = dateTimeVals[tokenPos];
+        Skip();
       }
 
@@ -349 +513 @@
         if (!reader.EndOfStream) {
           CurrentLine = reader.ReadLine();
-          var newTokens = from str in Split(CurrentLine)
-                          let trimmedStr = str.Trim()
-                          where !string.IsNullOrEmpty(trimmedStr)
-                          select MakeToken(trimmedStr);
-
-          tokens.AddRange(newTokens);
-          tokens.Add(NewlineToken);
           CurrentLineNumber++;
+          if (reader.BaseStream.CanSeek) {
+            BytesRead = reader.BaseStream.Position;
+          } else {
+            BytesRead += CurrentLine.Length + 2; // guess
+          }
+          int i = 0;
+          if (!string.IsNullOrWhiteSpace(CurrentLine)) {
+            foreach (var tok in Split(CurrentLine)) {
+              TokenTypeEnum type;
+              double doubleVal;
+              DateTime dateTimeValue;
+              type = TokenTypeEnum.String; // default
+              stringVals[i] = tok.Trim();
+              if (double.TryParse(tok, NumberStyles.Float, numberFormatInfo, out doubleVal)) {
+                type = TokenTypeEnum.Double;
+                doubleVals[i] = doubleVal;
+              } else if (DateTime.TryParse(tok, dateTimeFormatInfo, DateTimeStyles.None, out dateTimeValue)) {
+                type = TokenTypeEnum.DateTime;
+                dateTimeVals[i] = dateTimeValue;
+              } else if (string.IsNullOrWhiteSpace(tok)) {
+                type = TokenTypeEnum.Missing;
+              }
+
+              // couldn't parse the token as an int or float number or datetime value so return a string token
+
+              tokenTypes[i] = type;
+              i++;
+
+              if (i >= tokenTypes.Length) {
+                // increase buffer size if necessary
+                IncreaseCapacity(ref tokenTypes);
+                IncreaseCapacity(ref doubleVals);
+                IncreaseCapacity(ref stringVals);
+                IncreaseCapacity(ref dateTimeVals);
+              }
+            }
+          }
+          tokenTypes[i] = TokenTypeEnum.NewLine;
+          numTokens = i + 1;
+          tokenPos = 0;
         }
       }
 
       private IEnumerable<string> Split(string line) {
-        IEnumerable<string> splitString;
-        if (separator == WHITESPACECHAR) {
-          //separate whitespaces
-          splitString = line.Split(new char[0], StringSplitOptions.RemoveEmptyEntries);
-        } else {
-          splitString = line.Split(separator);
-        }
-        int cur = splitString.Count();
-        foreach (var str in splitString) {
-          yield return str;
-          cur--;
-          // do not return the INTERNAL_SEPARATOR after the last string
-          if (cur != 0) {
-            yield return INTERNAL_SEPARATOR;
-          }
-        }
-      }
-
-      private Token MakeToken(string strToken) {
-        Token token = new Token(TokenTypeEnum.String, strToken);
-        if (strToken.Equals(INTERNAL_SEPARATOR)) {
-          return SeparatorToken;
-        } else if (double.TryParse(strToken, NumberStyles.Float, numberFormatInfo, out token.doubleValue)) {
-          token.type = TokenTypeEnum.Double;
-          return token;
-        } else if (DateTime.TryParse(strToken, dateTimeFormatInfo, DateTimeStyles.None, out token.dateTimeValue)) {
-          token.type = TokenTypeEnum.DateTime;
-          return token;
-        }
-
-        // couldn't parse the token as an int or float number  or datetime value so return a string token
-        return token;
-      }
-
-      public Token Peek() {
-        return tokens[0];
-      }
-
-      public Token Next() {
-        Token next = tokens[0];
-        tokens.RemoveAt(0);
-        if (tokens.Count == 0) {
-          ReadNextTokens();
-        }
-        return next;
-      }
-
-      public bool HasNext() {
-        return tokens.Count > 0 || !reader.EndOfStream;
+        return separator == WHITESPACECHAR ?
+          line.Split(whiteSpaceSeparators, StringSplitOptions.RemoveEmptyEntries) :
+          line.Split(separators);
+      }
+
+      private static void IncreaseCapacity<T>(ref T[] arr) {
+        int n = (int)Math.Floor(arr.Length * 1.7); // guess
+        T[] arr2 = new T[n];
+        Array.Copy(arr, arr2, arr.Length);
+        arr = arr2;
       }
     }
@@ -415 +573 @@
 
     #region parsing
-    private void Parse(bool columnNamesInFirstLine) {
-      if (columnNamesInFirstLine) {
-        ParseVariableNames();
-        if (!tokenizer.HasNext())
-          Error(
-            "Couldn't parse data values. Probably because of incorrect number format (the parser expects english number format with a '.' as decimal separator).",
-            "", tokenizer.CurrentLineNumber);
-      }
-      ParseValues();
-      if (rowValues.Count == 0) Error("Couldn't parse data values. Probably because of incorrect number format (the parser expects english number format with a '.' as decimal separator).", "", tokenizer.CurrentLineNumber);
-    }
-
-    private void ParseValues() {
-      while (tokenizer.HasNext()) {
-        if (tokenizer.Peek() == tokenizer.NewlineToken) {
-          tokenizer.Next();
-        } else {
-          List<object> row = new List<object>();
-          object value = NextValue(tokenizer);
-          row.Add(value);
-          while (tokenizer.HasNext() && tokenizer.Peek() == tokenizer.SeparatorToken) {
-            Expect(tokenizer.SeparatorToken);
-            row.Add(NextValue(tokenizer));
-          }
-          Expect(tokenizer.NewlineToken);
-          // all rows have to have the same number of values            
-          // the first row defines how many samples are needed
-          if (rowValues.Count > 0 && rowValues[0].Count != row.Count) {
-            Error("The first row of the dataset has " + rowValues[0].Count + " columns." +
-                  "\nLine " + tokenizer.CurrentLineNumber + " has " + row.Count + " columns.", "",
-                  tokenizer.CurrentLineNumber);
-          }
-          rowValues.Add(row);
-        }
-      }
-    }
-
-    private object NextValue(Tokenizer tokenizer) {
-      if (tokenizer.Peek() == tokenizer.SeparatorToken || tokenizer.Peek() == tokenizer.NewlineToken) return string.Empty;
-      Token current = tokenizer.Next();
-      if (current.type == TokenTypeEnum.Separator) {
-        return double.NaN;
-      } else if (current.type == TokenTypeEnum.String) {
-        return current.stringValue;
-      } else if (current.type == TokenTypeEnum.Double) {
-        return current.doubleValue;
-      } else if (current.type == TokenTypeEnum.DateTime) {
-        return current.dateTimeValue;
-      }
-      // found an unexpected token => throw error 
-      Error("Unexpected token.", current.stringValue, tokenizer.CurrentLineNumber);
-      // this line is never executed because Error() throws an exception
-      throw new InvalidOperationException();
-    }
 
     private void ParseVariableNames() {
       // the first line must contain variable names
-      List<Token> tokens = new List<Token>();
-      Token valueToken;
-      valueToken = tokenizer.Next();
-      tokens.Add(valueToken);
-      while (tokenizer.HasNext() && tokenizer.Peek() == tokenizer.SeparatorToken) {
-        Expect(tokenizer.SeparatorToken);
-        valueToken = tokenizer.Next();
-        if (valueToken != tokenizer.NewlineToken) {
-          tokens.Add(valueToken);
-        }
-      }
-      if (valueToken != tokenizer.NewlineToken) {
-        Expect(tokenizer.NewlineToken);
-      }
-      variableNames = tokens.Select(x => x.stringValue.Trim()).ToList();
-    }
-
-    private void Expect(Token expectedToken) {
-      Token actualToken = tokenizer.Next();
-      if (actualToken != expectedToken) {
-        Error("Expected: " + expectedToken, actualToken.stringValue, tokenizer.CurrentLineNumber);
-      }
+      List<string> varNames = new List<string>();
+
+      TokenTypeEnum type;
+      string strVal;
+      double dblVal;
+      DateTime dateTimeVal;
+
+      tokenizer.Next(out type, out strVal, out dblVal, out dateTimeVal);
+
+      // the first token must be a variable name
+      if (type != TokenTypeEnum.String)
+        throw new ArgumentException("Error: Expected " + TokenTypeEnum.String + " got " + type);
+      varNames.Add(strVal);
+
+      while (tokenizer.HasNext() && tokenizer.PeekType() != TokenTypeEnum.NewLine) {
+        tokenizer.Next(out type, out strVal, out dblVal, out dateTimeVal);
+        varNames.Add(strVal);
+      }
+      ExpectType(TokenTypeEnum.NewLine);
+
+      variableNames = varNames;
+    }
+
+    private void ExpectType(TokenTypeEnum expectedToken) {
+      if (tokenizer.PeekType() != expectedToken)
+        throw new ArgumentException("Error: Expected " + expectedToken + " got " + tokenizer.PeekType());
+      tokenizer.Skip();
     }