Changeset 6740 for trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/TableFileParser.cs

Timestamp:

09/12/11 13:48:31 (13 years ago)

Author:

mkommend

Message:

#1597, #1609, #1640:

• Corrected TableFileParser to handle empty rows correctly.
• Refactored DataSet to store values in List<List> instead of a two-dimensional array.
• Enable importing and storing string and datetime values.
• Changed data access methods in dataset and adapted all concerning classes.
• Changed interpreter to store the variable values for all rows during the compilation step.

File:

• trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/TableFileParser.cs (modified) (18 diffs)

trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/TableFileParser.cs

@@ -21 +21 @@
 
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Globalization;
@@ -33 +34 @@
     private readonly char[] POSSIBLE_SEPARATORS = new char[] { ',', ';', '\t' };
     private Tokenizer tokenizer;
-    private List<List<double>> rowValues;
+    private List<List<object>> rowValues;
 
     private int rows;
@@ -47 +48 @@
     }
 
-    private double[,] values;
-    public double[,] Values {
+    private List<IList> values;
+    public List<IList> Values {
       get {
         return values;
@@ -69 +70 @@
 
     public TableFileParser() {
-      rowValues = new List<List<double>>();
+      rowValues = new List<List<object>>();
       variableNames = new List<string>();
     }
@@ -75 +76 @@
     public void Parse(string fileName) {
       NumberFormatInfo numberFormat;
+      DateTimeFormatInfo dateTimeFormatInfo;
       char separator;
-      DetermineFileFormat(fileName, out numberFormat, out separator);
+      DetermineFileFormat(fileName, out numberFormat, out dateTimeFormatInfo, out separator);
       using (StreamReader reader = new StreamReader(fileName)) {
-        tokenizer = new Tokenizer(reader, numberFormat, separator);
+        tokenizer = new Tokenizer(reader, numberFormat, dateTimeFormatInfo, separator);
         // parse the file
         Parse();
@@ -86 +88 @@
       rows = rowValues.Count;
       columns = rowValues[0].Count;
-      values = new double[rows, columns];
-
-      int rowIndex = 0;
-      int columnIndex = 0;
-      foreach (List<double> row in rowValues) {
-        columnIndex = 0;
-        foreach (double element in row) {
-          values[rowIndex, columnIndex++] = element;
-        }
-        rowIndex++;
-      }
-    }
-
-    private void DetermineFileFormat(string fileName, out NumberFormatInfo numberFormat, out char separator) {
+      values = new List<IList>();
+
+      //create columns
+      for (int col = 0; col < columns; col++) {
+        var types = rowValues.Select(r => r[col]).Where(v => v != null && v as string != string.Empty).Take(10).Select(v => v.GetType());
+        if (!types.Any()) {
+          values.Add(new List<string>());
+          continue;
+        }
+
+        var columnType = types.GroupBy(v => v).OrderBy(v => v).Last().Key;
+        if (columnType == typeof(double)) values.Add(new List<double>());
+        else if (columnType == typeof(DateTime)) values.Add(new List<DateTime>());
+        else if (columnType == typeof(string)) values.Add(new List<string>());
+        else throw new InvalidOperationException();
+      }
+
+
+
+      //fill with values
+      foreach (List<object> row in rowValues) {
+        int columnIndex = 0;
+        foreach (object element in row) {
+          //handle missing values with default values
+          if (element as string == string.Empty) {
+            if (values[columnIndex] is List<double>) values[columnIndex].Add(double.NaN);
+            else if (values[columnIndex] is List<DateTime>) values[columnIndex].Add(DateTime.MinValue);
+            else if (values[columnIndex] is List<string>) values[columnIndex].Add(string.Empty);
+            else throw new InvalidOperationException();
+          } else values[columnIndex].Add(element);
+          columnIndex++;
+        }
+      }
+    }
+
+    private void DetermineFileFormat(string fileName, out NumberFormatInfo numberFormat, out DateTimeFormatInfo dateTimeFormatInfo, out char separator) {
       using (StreamReader reader = new StreamReader(fileName)) {
         // skip first line
@@ -123 +147 @@
         if (OccurrencesOf(charCounts, '.') > 10) {
           numberFormat = NumberFormatInfo.InvariantInfo;
+          dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
           separator = POSSIBLE_SEPARATORS
             .Where(c => OccurrencesOf(charCounts, c) > 10)
@@ -139 +164 @@
             // English format (only integer values) with ',' as separator
             numberFormat = NumberFormatInfo.InvariantInfo;
+            dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
             separator = ',';
           } else {
@@ -144 +170 @@
             // German format (real values)
             numberFormat = NumberFormatInfo.GetInstance(new CultureInfo("de-DE"));
+            dateTimeFormatInfo = DateTimeFormatInfo.GetInstance(new CultureInfo("de-DE"));
             separator = POSSIBLE_SEPARATORS
               .Except(disallowedSeparators)
@@ -154 +181 @@
           // no points and no commas => English format
           numberFormat = NumberFormatInfo.InvariantInfo;
+          dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
           separator = POSSIBLE_SEPARATORS
             .Where(c => OccurrencesOf(charCounts, c) > 10)
@@ -169 +197 @@
     #region tokenizer
     internal enum TokenTypeEnum {
-      NewLine, Separator, String, Double
+      NewLine, Separator, String, Double, DateTime
     }
 
@@ -176 +204 @@
       public string stringValue;
       public double doubleValue;
+      public DateTime dateTimeValue;
 
       public Token(TokenTypeEnum type, string value) {
         this.type = type;
         stringValue = value;
+        dateTimeValue = DateTime.MinValue;
         doubleValue = 0.0;
       }
@@ -193 +223 @@
       private List<Token> tokens;
       private NumberFormatInfo numberFormatInfo;
+      private DateTimeFormatInfo dateTimeFormatInfo;
       private char separator;
       private const string INTERNAL_SEPARATOR = "#";
@@ -218 +249 @@
       }
 
-      public Tokenizer(StreamReader reader, NumberFormatInfo numberFormatInfo, char separator) {
+      public Tokenizer(StreamReader reader, NumberFormatInfo numberFormatInfo, DateTimeFormatInfo dateTimeFormatInfo, char separator) {
         this.reader = reader;
         this.numberFormatInfo = numberFormatInfo;
+        this.dateTimeFormatInfo = dateTimeFormatInfo;
         this.separator = separator;
         separatorToken = new Token(TokenTypeEnum.Separator, INTERNAL_SEPARATOR);
@@ -264 +296 @@
           token.type = TokenTypeEnum.Double;
           return token;
-        }
-
-        // couldn't parse the token as an int or float number so return a string token
+        } else if (DateTime.TryParse(strToken, out token.dateTimeValue)) {
+          token.type = TokenTypeEnum.DateTime;
+          return token;
+        }
+
+        // couldn't parse the token as an int or float number  or datetime value so return a string token
         return token;
       }
@@ -299 +334 @@
     private void ParseValues() {
       while (tokenizer.HasNext()) {
-        List<double> row = new List<double>();
-        row.Add(NextValue(tokenizer));
+        List<object> row = new List<object>();
+        object value = NextValue(tokenizer);
+        if (value == null) { tokenizer.Next(); continue; }
+        row.Add(value);
         while (tokenizer.HasNext() && tokenizer.Peek() == tokenizer.SeparatorToken) {
           Expect(tokenizer.SeparatorToken);
@@ -312 +349 @@
             "\nLine " + tokenizer.CurrentLineNumber + " has " + row.Count + " columns.", "", tokenizer.CurrentLineNumber);
         }
-        // add the current row to the collection of rows and start a new row
         rowValues.Add(row);
-        row = new List<double>();
-      }
-    }
-
-    private double NextValue(Tokenizer tokenizer) {
-      if (tokenizer.Peek() == tokenizer.SeparatorToken || tokenizer.Peek() == tokenizer.NewlineToken) return double.NaN;
+        row = new List<object>();
+      }
+    }
+
+    private object NextValue(Tokenizer tokenizer) {
+      if (tokenizer.Peek() == tokenizer.SeparatorToken) return string.Empty;
+      if (tokenizer.Peek() == tokenizer.NewlineToken) return null;
       Token current = tokenizer.Next();
-      if (current.type == TokenTypeEnum.Separator || current.type == TokenTypeEnum.String) {
+      if (current.type == TokenTypeEnum.Separator) {
         return double.NaN;
+      } else if (current.type == TokenTypeEnum.String) {
+        return current.stringValue;
       } else if (current.type == TokenTypeEnum.Double) {
-        // just take the value
         return current.doubleValue;
+      } else if (current.type == TokenTypeEnum.DateTime) {
+        return current.dateTimeValue;
       }
       // found an unexpected token => throw error 
@@ -334 +374 @@
 
     private void ParseVariableNames() {
-      // if the first line doesn't start with a double value then we assume that the
-      // first line contains variable names
-      if (tokenizer.HasNext() && tokenizer.Peek().type != TokenTypeEnum.Double) {
-
-        List<Token> tokens = new List<Token>();
-        Token valueToken;
+      //if first token is double no variables names are given
+      if (tokenizer.Peek().type == TokenTypeEnum.Double) return;
+
+      // the first line must contain variable names
+      List<Token> tokens = new List<Token>();
+      Token valueToken;
+      valueToken = tokenizer.Next();
+      tokens.Add(valueToken);
+      while (tokenizer.HasNext() && tokenizer.Peek() == tokenizer.SeparatorToken) {
+        Expect(tokenizer.SeparatorToken);
         valueToken = tokenizer.Next();
-        tokens.Add(valueToken);
-        while (tokenizer.HasNext() && tokenizer.Peek() == tokenizer.SeparatorToken) {
-          Expect(tokenizer.SeparatorToken);
-          valueToken = tokenizer.Next();
-          if (valueToken != tokenizer.NewlineToken) {
-            tokens.Add(valueToken);
-          }
-        }
         if (valueToken != tokenizer.NewlineToken) {
-          Expect(tokenizer.NewlineToken);
-        }
-        variableNames = tokens.Select(x => x.stringValue.Trim()).ToList();
-      }
+          tokens.Add(valueToken);
+        }
+      }
+      if (valueToken != tokenizer.NewlineToken) {
+        Expect(tokenizer.NewlineToken);
+      }
+      variableNames = tokens.Select(x => x.stringValue.Trim()).ToList();
     }