Changeset 17414 for branches/3040_VectorBasedGP/HeuristicLab.Problems.Instances.DataAnalysis/3.3/TableFileParser.cs

Timestamp:

01/31/20 12:42:47 (4 years ago)

Author:

pfleck

Message:

#3040 Started adding UCI time series regression benchmarks.
Adapted parser (extracted format options & added parsing for double vectors).

File:

• branches/3040_VectorBasedGP/HeuristicLab.Problems.Instances.DataAnalysis/3.3/TableFileParser.cs (modified) (14 diffs)

branches/3040_VectorBasedGP/HeuristicLab.Problems.Instances.DataAnalysis/3.3/TableFileParser.cs

@@ -20 +20 @@
 #endregion
 
-
 using System;
 using System.Collections;
@@ -29 +28 @@
 using System.Linq;
 using System.Text;
+using HeuristicLab.Problems.DataAnalysis;
 
 namespace HeuristicLab.Problems.Instances.DataAnalysis {
@@ -89 +89 @@
 
     public bool AreColumnNamesInFirstLine(string fileName) {
-      NumberFormatInfo numberFormat;
-      DateTimeFormatInfo dateTimeFormatInfo;
-      char separator;
-      DetermineFileFormat(fileName, out numberFormat, out dateTimeFormatInfo, out separator);
+      var formatOptions = DetermineFileFormat(fileName);
       using (var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
-        return AreColumnNamesInFirstLine(stream, numberFormat, dateTimeFormatInfo, separator);
+        return AreColumnNamesInFirstLine(stream, formatOptions);
       }
     }
 
     public bool AreColumnNamesInFirstLine(Stream stream) {
-      NumberFormatInfo numberFormat = NumberFormatInfo.InvariantInfo;
-      DateTimeFormatInfo dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
-      char separator = ',';
-      return AreColumnNamesInFirstLine(stream, numberFormat, dateTimeFormatInfo, separator);
-    }
-
-    public bool AreColumnNamesInFirstLine(string fileName, NumberFormatInfo numberFormat,
-                                         DateTimeFormatInfo dateTimeFormatInfo, char separator) {
+      var formatOptions = new TableFileFormatOptions {
+        NumberFormat = NumberFormatInfo.InvariantInfo,
+        DateTimeFormat = DateTimeFormatInfo.InvariantInfo,
+        ColumnSeparator = ','
+      };
+      return AreColumnNamesInFirstLine(stream, formatOptions);
+    }
+
+    public bool AreColumnNamesInFirstLine(string fileName, TableFileFormatOptions formatOptions) {
       using (var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
-        return AreColumnNamesInFirstLine(stream, numberFormat, dateTimeFormatInfo, separator);
-      }
-    }
-
-    public bool AreColumnNamesInFirstLine(Stream stream, NumberFormatInfo numberFormat,
-                                          DateTimeFormatInfo dateTimeFormatInfo, char separator) {
+        return AreColumnNamesInFirstLine(stream, formatOptions);
+      }
+    }
+
+    public bool AreColumnNamesInFirstLine(Stream stream, TableFileFormatOptions formatOptions) {
       using (StreamReader reader = new StreamReader(stream, Encoding)) {
-        tokenizer = new Tokenizer(reader, numberFormat, dateTimeFormatInfo, separator);
+        tokenizer = new Tokenizer(reader, formatOptions);
         return (tokenizer.PeekType() != TokenTypeEnum.Double);
       }
@@ -126 +123 @@
     /// <param name="columnNamesInFirstLine"></param>
     public void Parse(string fileName, bool columnNamesInFirstLine, int lineLimit = -1) {
-      NumberFormatInfo numberFormat;
-      DateTimeFormatInfo dateTimeFormatInfo;
-      char separator;
-      DetermineFileFormat(fileName, out numberFormat, out dateTimeFormatInfo, out separator);
+      var formatOptions = DetermineFileFormat(fileName);
       EstimateNumberOfLines(fileName);
-      Parse(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine, lineLimit);
+      Parse(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), formatOptions, columnNamesInFirstLine, lineLimit);
     }
 
@@ -142 +136 @@
     /// <param name="separator">defines the separator</param>
     /// <param name="columnNamesInFirstLine"></param>
-    public void Parse(string fileName, NumberFormatInfo numberFormat, DateTimeFormatInfo dateTimeFormatInfo, char separator, bool columnNamesInFirstLine, int lineLimit = -1) {
+    public void Parse(string fileName, TableFileFormatOptions formatOptions, bool columnNamesInFirstLine, int lineLimit = -1) {
       EstimateNumberOfLines(fileName);
       using (var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
-        Parse(stream, numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine, lineLimit);
+        Parse(stream, formatOptions, columnNamesInFirstLine, lineLimit);
       }
     }
@@ -182 +176 @@
     /// <param name="columnNamesInFirstLine"></param>
     public void Parse(Stream stream, bool columnNamesInFirstLine, int lineLimit = -1) {
-      NumberFormatInfo numberFormat = NumberFormatInfo.InvariantInfo;
-      DateTimeFormatInfo dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
-      char separator = ',';
-      Parse(stream, numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine, lineLimit);
+      var formatOptions = new TableFileFormatOptions {
+        NumberFormat = NumberFormatInfo.InvariantInfo,
+        DateTimeFormat = DateTimeFormatInfo.InvariantInfo,
+        ColumnSeparator = ','
+      };
+      Parse(stream, formatOptions, columnNamesInFirstLine, lineLimit);
     }
 
@@ -196 +192 @@
     /// <param name="separator">defines the separator</param>
     /// <param name="columnNamesInFirstLine"></param>
-    public void Parse(Stream stream, NumberFormatInfo numberFormat, DateTimeFormatInfo dateTimeFormatInfo, char separator, bool columnNamesInFirstLine, int lineLimit = -1) {
+    public void Parse(Stream stream, TableFileFormatOptions formatOptions, bool columnNamesInFirstLine, int lineLimit = -1) {
       if (lineLimit > 0) estimatedNumberOfLines = lineLimit;
 
       using (var reader = new StreamReader(stream)) {
-        tokenizer = new Tokenizer(reader, numberFormat, dateTimeFormatInfo, separator);
+        tokenizer = new Tokenizer(reader, formatOptions);
         var strValues = new List<List<string>>();
         values = new List<IList>();
@@ -257 +253 @@
       this.columns = values.Count;
 
+      // see if any string column can be converted to vectors
+      if (formatOptions.VectorSeparator != null) {
+        for (int i = 0; i < values.Count; i++) {
+          if (!(values[i] is List<string> stringList)) continue;
+
+          var strings = new string[stringList.Count][];
+          var doubles = new double[strings.Length][];
+          bool allDoubles = true;
+          for (int j = 0; j < strings.Length && allDoubles; j++) {
+            strings[j] = stringList[j].Split(formatOptions.VectorSeparator.Value);
+            doubles[j] = new double[strings[j].Length];
+            for (int k = 0; k < doubles[j].Length && allDoubles; k++) {
+              allDoubles = double.TryParse(strings[j][k], NumberStyles.Float, formatOptions.NumberFormat, out doubles[j][k]);
+            }
+          }
+
+          if (allDoubles) {
+            var vectorList = new List<DoubleVector>(stringList.Count);
+            for (int j = 0; j < doubles.Length; j++) {
+              vectorList.Add(new DoubleVector(doubles[j]));
+            }
+
+            values[i] = vectorList;
+          }
+        }
+      }
+
       // replace lists with undefined type (object) with double-lists
       for (int i = 0; i < values.Count; i++) {
@@ -271 +294 @@
         var stringList = l as List<string>;
         var objList = l as List<object>;
+        var vecList = l as List<DoubleVector>;
         if (dblList != null) dblList.TrimExcess();
         if (byteList != null) byteList.TrimExcess();
@@ -276 +300 @@
         if (stringList != null) stringList.TrimExcess();
         if (objList != null) objList.TrimExcess();
+        if (vecList != null) vecList.TrimExcess();
       }
 
@@ -422 +447 @@
     #endregion
 
-    public static void DetermineFileFormat(string path, out NumberFormatInfo numberFormat, out DateTimeFormatInfo dateTimeFormatInfo, out char separator) {
-      DetermineFileFormat(new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), out numberFormat, out dateTimeFormatInfo, out separator);
-    }
-
-    public static void DetermineFileFormat(Stream stream, out NumberFormatInfo numberFormat, out DateTimeFormatInfo dateTimeFormatInfo, out char separator) {
+    public static TableFileFormatOptions DetermineFileFormat(string path) {
+      return DetermineFileFormat(new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite));
+    }
+
+    public static TableFileFormatOptions DetermineFileFormat(Stream stream) {
       using (StreamReader reader = new StreamReader(stream)) {
         // skip first line
@@ -449 +474 @@
         // in all cases only treat ' ' as separator if no other separator is possible (spaces can also occur additionally to separators)
         if (OccurrencesOf(charCounts, '.') > 10) {
-          numberFormat = NumberFormatInfo.InvariantInfo;
-          dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
-          separator = POSSIBLE_SEPARATORS
-            .Where(c => OccurrencesOf(charCounts, c) > 10)
-            .OrderBy(c => -OccurrencesOf(charCounts, c))
-            .DefaultIfEmpty(' ')
-            .First();
+          return new TableFileFormatOptions {
+            NumberFormat = NumberFormatInfo.InvariantInfo,
+            DateTimeFormat = DateTimeFormatInfo.InvariantInfo,
+            ColumnSeparator = POSSIBLE_SEPARATORS
+              .Where(c => OccurrencesOf(charCounts, c) > 10)
+              .OrderBy(c => -OccurrencesOf(charCounts, c))
+                .DefaultIfEmpty(' ')
+              .First()
+          };
         } else if (OccurrencesOf(charCounts, ',') > 10) {
           // no points and many commas
@@ -470 +497 @@
           if (tokensWithMultipleCommas > 1) {
             // English format (only integer values) with ',' as separator
-            numberFormat = NumberFormatInfo.InvariantInfo;
-            dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
-            separator = ',';
+            return new TableFileFormatOptions {
+              NumberFormat = NumberFormatInfo.InvariantInfo,
+              DateTimeFormat = DateTimeFormatInfo.InvariantInfo,
+              ColumnSeparator = ','
+            };
           } else {
             char[] disallowedSeparators = new char[] { ',' }; // n. def. contains a space so ' ' should be disallowed to, however existing unit tests would fail
             // German format (real values)
-            numberFormat = NumberFormatInfo.GetInstance(new CultureInfo("de-DE"));
-            dateTimeFormatInfo = DateTimeFormatInfo.GetInstance(new CultureInfo("de-DE"));
-            separator = POSSIBLE_SEPARATORS
-              .Except(disallowedSeparators)
+            return new TableFileFormatOptions {
+              NumberFormat = NumberFormatInfo.GetInstance(new CultureInfo("de-DE")),
+              DateTimeFormat = DateTimeFormatInfo.GetInstance(new CultureInfo("de-DE")),
+              ColumnSeparator = POSSIBLE_SEPARATORS
+                .Except(disallowedSeparators)
+                .Where(c => OccurrencesOf(charCounts, c) > 10)
+                .OrderBy(c => -OccurrencesOf(charCounts, c))
+                .DefaultIfEmpty(' ')
+                .First()
+            };
+          }
+        } else {
+          // no points and no commas => English format
+          return new TableFileFormatOptions {
+            NumberFormat = NumberFormatInfo.InvariantInfo,
+            DateTimeFormat = DateTimeFormatInfo.InvariantInfo,
+            ColumnSeparator = POSSIBLE_SEPARATORS
               .Where(c => OccurrencesOf(charCounts, c) > 10)
               .OrderBy(c => -OccurrencesOf(charCounts, c))
               .DefaultIfEmpty(' ')
-              .First();
-          }
-        } else {
-          // no points and no commas => English format
-          numberFormat = NumberFormatInfo.InvariantInfo;
-          dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
-          separator = POSSIBLE_SEPARATORS
-            .Where(c => OccurrencesOf(charCounts, c) > 10)
-            .OrderBy(c => -OccurrencesOf(charCounts, c))
-            .DefaultIfEmpty(' ')
-            .First();
+              .First()
+          };
         }
       }
@@ -540 +573 @@
       }
 
-      public Tokenizer(StreamReader reader, NumberFormatInfo numberFormatInfo, DateTimeFormatInfo dateTimeFormatInfo, char separator) {
+      public Tokenizer(StreamReader reader, TableFileFormatOptions formatOptions) {
         this.reader = reader;
-        this.numberFormatInfo = numberFormatInfo;
-        this.dateTimeFormatInfo = dateTimeFormatInfo;
-        this.separator = separator;
+        this.numberFormatInfo = formatOptions.NumberFormat;
+        this.dateTimeFormatInfo = formatOptions.DateTimeFormat;
+        this.separator = formatOptions.ColumnSeparator;
         this.separators = new char[] { separator };
         ReadNextTokens();