Index: trunk/sources/HeuristicLab.Tests/HeuristicLab.Problems.Instances.DataAnalysis-3.3/TableFileParserTest.cs
===================================================================
--- trunk/sources/HeuristicLab.Tests/HeuristicLab.Problems.Instances.DataAnalysis-3.3/TableFileParserTest.cs	(revision 14282)
+++ trunk/sources/HeuristicLab.Tests/HeuristicLab.Problems.Instances.DataAnalysis-3.3/TableFileParserTest.cs	(revision 14284)
@@ -24,4 +24,5 @@
 using System.Globalization;
 using System.IO;
+using System.Text;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 
@@ -47,5 +48,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual(parser.Values[3][0], 3.14);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -71,5 +73,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual(parser.Values[3][0], 3.14);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -94,5 +97,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual(parser.Values[3][0], 3.14);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -118,5 +122,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual(parser.Values[3][0], 3.14);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -141,5 +146,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual((double)parser.Values[3][0], 3);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -165,5 +171,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual((double)parser.Values[3][0], 3);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -188,5 +195,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual((double)parser.Values[3][0], 3);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -211,5 +219,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual((double)parser.Values[3][0], 3);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -235,5 +244,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual((double)parser.Values[3][0], 3);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -259,5 +269,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual((double)parser.Values[3][0], 3);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -283,5 +294,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual((double)parser.Values[3][0], 3.14);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -307,5 +319,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual((double)parser.Values[3][0], 3.14);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -330,5 +343,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual((double)parser.Values[3][0], 3.14);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -354,5 +368,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual((double)parser.Values[3][0], 3.14);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -377,5 +392,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual((double)parser.Values[3][0], 3);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -401,5 +417,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual((double)parser.Values[3][0], 3);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -424,5 +441,6 @@
         Assert.AreEqual(4, parser.Rows);
         Assert.AreEqual(4, parser.Columns);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -447,5 +465,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual((double)parser.Values[3][0], 3.14);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -471,5 +490,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual((double)parser.Values[3][0], 3.14);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -494,5 +514,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual((double)parser.Values[3][0], 3.14);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -518,5 +539,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual((double)parser.Values[3][0], 3.14);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -539,5 +561,6 @@
         Assert.AreEqual(3, parser.Rows);
         Assert.AreEqual(4507, parser.Columns);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -562,5 +585,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual((double)parser.Values[3][0], 3);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -586,5 +610,6 @@
         Assert.AreEqual(4, parser.Columns);
         Assert.AreEqual((double)parser.Values[3][0], 3);
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -627,5 +652,6 @@
         Assert.IsTrue(double.IsPositiveInfinity((double)parser.Values[1][3])); // NOTE: in DE-DE NumberFormat just "unendlich" is not allowed (compare with InvariantCulture)
         Assert.IsTrue(double.IsNegativeInfinity((double)parser.Values[1][4]));
-      } finally {
+      }
+      finally {
         File.Delete(tempFileName);
       }
@@ -664,6 +690,241 @@
         Assert.IsTrue(double.IsPositiveInfinity((double)parser.Values[1][3])); // NOTE: in InvariantCulture +Infinity is not allowed (compare with DE-DE)
         Assert.IsTrue(double.IsNegativeInfinity((double)parser.Values[1][4]));
-      } finally {
-        File.Delete(tempFileName);
+      }
+      finally {
+        File.Delete(tempFileName);
+      }
+    }
+
+
+    [TestMethod]
+    [TestCategory("Problems.Instances")]
+    [TestProperty("Time", "short")]
+    public void ParseWithTypeConversion() {
+      // the parser tries to determine the column type (double, datetime, string) by looking at the values in the first few rows
+      // if the values are of a different type then the type of the column is converted
+      {
+        // case 1 
+        // default for values is double and therefore a column with all missing values should be List<double> and contain NaN
+        var tmpFileName = Path.GetTempFileName();
+        WriteToFile(tmpFileName,
+          @"stringCol,note
+,missing val
+3.14,double
+");
+
+        TableFileParser parser = new TableFileParser();
+        try {
+          parser.Parse(tmpFileName,
+            CultureInfo.InvariantCulture.NumberFormat,
+            CultureInfo.InvariantCulture.DateTimeFormat,
+            separator: ',', columnNamesInFirstLine: true);
+          Assert.IsTrue(parser.Values[0] is List<double>);
+          Assert.AreEqual(double.NaN, parser.Values[0][0]);
+          Assert.AreEqual(3.14, parser.Values[0][1]);
+        }
+        finally {
+          File.Delete(tmpFileName);
+        }
+
+      }
+
+      {
+        // case 2
+        // 'The first missing values are replaced with double.NaN while the last ones with string.Empty.'
+
+        var tmpFileName = Path.GetTempFileName();
+        WriteToFile(tmpFileName,
+          @"stringCol,note
+,missing val
+3.14,
+,missing val
+str,a string --> column is converted to List<string>
+,missing val
+");
+
+        TableFileParser parser = new TableFileParser();
+        try {
+          parser.Parse(tmpFileName,
+            CultureInfo.InvariantCulture.NumberFormat,
+            CultureInfo.InvariantCulture.DateTimeFormat,
+            separator: ',', columnNamesInFirstLine: true);
+          Assert.IsTrue(parser.Values[0] is List<string>);
+          Assert.AreEqual(string.Empty, parser.Values[0][0]);
+          Assert.AreEqual("3.14", parser.Values[0][1]);
+          Assert.AreEqual(string.Empty, parser.Values[0][2]);
+          Assert.AreEqual("str", parser.Values[0][3]);
+          Assert.AreEqual(string.Empty, parser.Values[0][4]);
+        }
+        finally {
+          File.Delete(tmpFileName);
+        }
+      }
+
+      {
+        // case 3
+        // DateTime conversion to strings
+        var tmpFileName = Path.GetTempFileName();
+        WriteToFile(tmpFileName,
+          @"stringCol,note
+,missing val
+3.1.2016,
+,missing val
+str,a string --> column is converted to List<string>
+,missing val
+");
+
+        TableFileParser parser = new TableFileParser();
+        try {
+          parser.Parse(tmpFileName,
+            CultureInfo.InvariantCulture.NumberFormat,
+            CultureInfo.InvariantCulture.DateTimeFormat,
+            separator: ',', columnNamesInFirstLine: true);
+          Assert.IsTrue(parser.Values[0] is List<string>);
+          Assert.AreEqual(string.Empty, parser.Values[0][0]);
+          Assert.AreEqual("3.1.2016", parser.Values[0][1]);
+          Assert.AreEqual(string.Empty, parser.Values[0][2]);
+          Assert.AreEqual("str", parser.Values[0][3]);
+          Assert.AreEqual(string.Empty, parser.Values[0][4]);
+        }
+        finally {
+          File.Delete(tmpFileName);
+        }
+      }
+    }
+
+    [TestMethod]
+    [TestCategory("Problems.Instances")]
+    [TestProperty("Time", "short")]
+    public void ParseDateTime() {
+      {
+        // case 1 dates and datetimes should be parsed as datetime column
+        var tmpFileName = Path.GetTempFileName();
+        WriteToFile(tmpFileName,
+          @"stringCol,note
+19.6.2016,date
+19.6.2016 8:15,datetime 
+");
+
+        TableFileParser parser = new TableFileParser();
+        try {
+          parser.Parse(tmpFileName,
+            CultureInfo.GetCultureInfo("de-de").NumberFormat,
+            CultureInfo.GetCultureInfo("de-de").DateTimeFormat,
+            separator: ',', columnNamesInFirstLine: true);
+          Assert.IsTrue(parser.Values[0] is List<DateTime>);
+          Assert.AreEqual(new DateTime(2016, 6, 19), parser.Values[0][0]);
+          Assert.AreEqual(new DateTime(2016, 6, 19, 8, 15, 0), parser.Values[0][1]);
+
+          WriteToFile(tmpFileName,
+            @"stringCol,note
+2016/6/19,date
+2016/6/19 8:15,datetime 
+");
+
+          parser.Parse(tmpFileName,
+            CultureInfo.InvariantCulture.NumberFormat,
+            CultureInfo.InvariantCulture.DateTimeFormat,
+            separator: ',', columnNamesInFirstLine: true);
+          Assert.IsTrue(parser.Values[0] is List<DateTime>);
+          Assert.AreEqual(new DateTime(2016, 6, 19), parser.Values[0][0]);
+          Assert.AreEqual(new DateTime(2016, 6, 19, 8, 15, 0), parser.Values[0][1]);
+        }
+
+        finally {
+          File.Delete(tmpFileName);
+        }
+      }
+
+      {
+        // case 2 never parse time values as datetimes
+        var tmpFileName = Path.GetTempFileName();
+        WriteToFile(tmpFileName,
+          @"stringCol,note
+8:15,time value
+9:40,time value
+");
+
+        TableFileParser parser = new TableFileParser();
+        try {
+          parser.Parse(tmpFileName,
+            CultureInfo.InvariantCulture.NumberFormat,
+            CultureInfo.InvariantCulture.DateTimeFormat,
+            separator: ',', columnNamesInFirstLine: true);
+          Assert.IsTrue(parser.Values[0] is List<string>); // time values should be parsed as strings
+          Assert.AreEqual("8:15", parser.Values[0][0]);
+          Assert.AreEqual("9:40", parser.Values[0][1]);
+        }
+        finally {
+          File.Delete(tmpFileName);
+        }
+      }
+    }
+
+
+
+    [TestMethod]
+    [TestCategory("Problems.Instances")]
+    [TestProperty("Time", "short")]
+    public void CheckTypeConversionAndLongFiles() {
+      {
+        // case 1 incorrect input after 500 rows should lead to exceptions
+        var tmpFileName = Path.GetTempFileName();
+        // create input data
+        var sb = new StringBuilder();
+        sb.AppendLine("col1,col2");
+        for (int r = 0; r < 2000; r++) {
+          sb.AppendLine("3.15, 3.15");
+        }
+        // add a row with only one input value
+        sb.AppendLine("3.15");
+
+        WriteToFile(tmpFileName, sb.ToString());
+
+        TableFileParser parser = new TableFileParser();
+        try {
+          parser.Parse(tmpFileName,
+            CultureInfo.InvariantCulture.NumberFormat,
+            CultureInfo.InvariantCulture.DateTimeFormat,
+            separator: ',', columnNamesInFirstLine: true);
+          // Parse should fail with an exception
+          Assert.Fail("expected exception TableFileParser.DataFormatException");
+        }
+        catch (TableFileParser.DataFormatException) {
+          // ignore the expected exception
+        }
+
+        finally {
+          File.Delete(tmpFileName);
+        }
+      }
+      {
+        // case 2
+        var tmpFileName = Path.GetTempFileName();
+        // create input data
+        var sb = new StringBuilder();
+        sb.AppendLine("doubleCol,stringCol");
+        for (int r = 0; r < 2000; r++) {
+          sb.AppendLine("3.15, 3.15");
+        }
+        // add a row with a string value --> the column should be converted to string
+        sb.AppendLine("3.15,str");
+
+        WriteToFile(tmpFileName, sb.ToString());
+
+        TableFileParser parser = new TableFileParser();
+        try {
+          parser.Parse(tmpFileName,
+            CultureInfo.InvariantCulture.NumberFormat,
+            CultureInfo.InvariantCulture.DateTimeFormat,
+            separator: ',', columnNamesInFirstLine: true);
+          Assert.IsTrue(parser.Values[0] is List<double>);
+          Assert.IsTrue(parser.Values[1] is List<string>);
+          Assert.AreEqual(parser.Values[1][0], "3.15");
+          Assert.AreEqual(parser.Values[1][2000], "str");
+        }
+
+        finally {
+          File.Delete(tmpFileName);
+        }
       }
     }
