Changeset 13526 for trunk

Timestamp:

01/16/16 15:22:52 (9 years ago)

Author:

gkronber

Message:

#2071 added code for type conversion of columns to the table file parser and made some other minor changes

File:

• trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/TableFileParser.cs (modified) (9 diffs)

trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/TableFileParser.cs

@@ -24 +24 @@
 using System.Collections;
 using System.Collections.Generic;
+using System.Diagnostics.Contracts;
 using System.Globalization;
 using System.IO;
@@ -141 +142 @@
     private void EstimateNumberOfLines(string fileName) {
       var len = new System.IO.FileInfo(fileName).Length;
-      var buf = new char[64 * 1024];
+      var buf = new char[1024 * 1024];
       using (var reader = new StreamReader(fileName)) {
         reader.ReadBlock(buf, 0, buf.Length);
@@ -233 +234 @@
                 tokenizer.CurrentLineNumber);
             }
+            if (!IsColumnTypeCompatible(values[colIdx], type)) {
+              values[colIdx] = ConvertToStringColumn(values[colIdx]);
+            }
             // add the value to the column
             AddValue(type, values[colIdx++], strVal, dblVal, dateTimeVal);
@@ -265 +269 @@
 
     #region type-dependent dispatch
+    private bool IsColumnTypeCompatible(IList list, TokenTypeEnum tokenType) {
+      return (list is List<string>) || // all tokens can be added to a string list
+             (tokenType == TokenTypeEnum.Missing) || // empty entries are allowed in all columns
+             (tokenType == TokenTypeEnum.Double && list is List<double>) ||
+             (tokenType == TokenTypeEnum.DateTime && list is List<DateTime>);
+    }
+
+    // all columns are converted to string columns when we find an non-empty value that has incorrect type
+    private IList ConvertToStringColumn(IList list) {
+      var dblL = list as List<double>;
+      if (dblL != null) {
+        var l = new List<string>(dblL.Capacity);
+        l.AddRange(dblL.Select(dbl => dbl.ToString()));
+        return l;
+      }
+
+      var dtL = list as List<DateTime>;
+      if (dtL != null) {
+        var l = new List<string>(dtL.Capacity);
+        l.AddRange(dtL.Select(dbl => dbl.ToString()));
+        return l;
+      }
+
+      if (list is List<string>) return list;
+
+      throw new InvalidProgramException(string.Format("Cannot convert column of type {0} to string column", list.GetType()));
+    }
+
     private void AddValue(TokenTypeEnum type, IList list, string strVal, double dblVal, DateTime dateTimeVal) {
-      switch (type) {
-        case TokenTypeEnum.Double:
-          AddDoubleToList(list, dblVal);
-          break;
-        case TokenTypeEnum.String:
-          AddStringToList(list, strVal);
-          break;
-        case TokenTypeEnum.DateTime:
-          AddDateTimeToList(list, dateTimeVal);
-          break;
-        default:
-          throw new InvalidOperationException();
-      }
-    }
-
-    private void AddDoubleToList(IList list, double dblVal) {
       var dblList = list as List<double>;
-      if (dblList != null) dblList.Add(dblVal);
-      else {
-        var strList = list as List<string>;
-        if (strList != null) strList.Add(dblVal.ToString());
-        else list.Add(null);
-      }
-    }
-
-    private void AddStringToList(IList list, string strVal) {
+      if (dblList != null) {
+        AddValue(type, dblList, dblVal);
+        return;
+      }
+
       var strList = list as List<string>;
-      if (strList != null) strList.Add(strVal);
-      else {
-        var dblList = list as List<double>;
-        if (dblList != null) dblList.Add(double.NaN);
-        else list.Add(null);
-      }
-    }
-
-    private void AddDateTimeToList(IList list, DateTime dateTimeVal) {
-      var dateTimeList = list as List<DateTime>;
-      if (dateTimeList != null) dateTimeList.Add(dateTimeVal);
-      else {
-        var dblList = list as List<double>;
-        if (dblList != null) dblList.Add(double.NaN);
-        else {
-          var strList = list as List<string>;
-          if (strList != null) strList.Add(dateTimeVal.ToString());
-          else list.Add(null);
-        }
-      }
+      if (strList != null) {
+        AddValue(type, strList, strVal);
+        return;
+      }
+      var dtList = list as List<DateTime>;
+      if (dtList != null) {
+        AddValue(type, dtList, dateTimeVal);
+        return;
+      }
+
+      list.Add(strVal); // assumes List<object>
+    }
+
+    private void AddValue(TokenTypeEnum type, List<double> list, double dblVal) {
+      Contract.Assert(type == TokenTypeEnum.Missing || type == TokenTypeEnum.Double);
+      list.Add(type == TokenTypeEnum.Missing ? double.NaN : dblVal);
+    }
+
+    private void AddValue(TokenTypeEnum type, List<string> list, string strVal) {
+      // assumes that strVal is always set to the original token read from the input file
+      list.Add(type == TokenTypeEnum.Missing ? string.Empty : strVal);
+    }
+
+    private void AddValue(TokenTypeEnum type, List<DateTime> list, DateTime dtVal) {
+      Contract.Assert(type == TokenTypeEnum.Missing || type == TokenTypeEnum.DateTime);
+      list.Add(type == TokenTypeEnum.Missing ? DateTime.MinValue : dtVal);
     }
 
@@ -320 +338 @@
           return new List<string>(estimatedNumberOfLines);
         case TokenTypeEnum.Double:
+        case TokenTypeEnum.Missing: // assume double columns
           return new List<double>(estimatedNumberOfLines);
         case TokenTypeEnum.DateTime:
@@ -381 +400 @@
             separator = ',';
           } else {
-            char[] disallowedSeparators = new char[] { ',' };
+            char[] disallowedSeparators = new char[] { ',' }; // n. def. contains a space so ' ' should be disallowed to, however existing unit tests would fail
             // German format (real values)
             numberFormat = NumberFormatInfo.GetInstance(new CultureInfo("de-DE"));
@@ -389 +408 @@
               .Where(c => OccurrencesOf(charCounts, c) > 10)
               .OrderBy(c => -OccurrencesOf(charCounts, c))
-              .DefaultIfEmpty(' ')
+              .DefaultIfEmpty(' ') 
               .First();
           }
@@ -412 +431 @@
     // the tokenizer reads full lines and returns separated tokens in the line as well as a terminating end-of-line character
     internal enum TokenTypeEnum {
-      NewLine, String, Double, DateTime
+      NewLine, String, Double, DateTime, Missing
     }
 
@@ -505 +524 @@
                 type = TokenTypeEnum.DateTime;
                 dateTimeVals[i] = dateTimeValue;
+              } else if (string.IsNullOrWhiteSpace(tok)) {
+                type = TokenTypeEnum.Missing;
               }