Changeset 13413

Timestamp:

11/29/15 15:33:15 (8 years ago)

Author:

gkronber

Message:

#2071: only preview first 500 lines of data in CSV import dialog.

Location:

trunk/sources

Files:

• HeuristicLab.Problems.Instances.DataAnalysis.Views/3.3/DataAnalysisImportTypeDialog.cs (modified) (1 diff)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/TableFileParser.cs (modified) (11 diffs)

trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis.Views/3.3/DataAnalysisImportTypeDialog.cs

@@ -129 +129 @@
                         (DateTimeFormatInfo)DateTimeFormatComboBox.SelectedValue,
                         (char)SeparatorComboBox.SelectedValue,
-                        CheckboxColumnNames.Checked);
+                        CheckboxColumnNames.Checked, lineLimit: 500);
         IEnumerable<string> variableNamesWithType = GetVariableNamesWithType(csvParser);
         PreviewDatasetMatrix.Content = new Dataset(variableNamesWithType, csvParser.Values);

trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/TableFileParser.cs

@@ -28 +28 @@
 using System.Linq;
 using System.Runtime.Serialization;
-using System.Security.Policy;
 
 namespace HeuristicLab.Problems.Instances.DataAnalysis {
@@ -114 +113 @@
     /// <param name="fileName">file which is parsed</param>
     /// <param name="columnNamesInFirstLine"></param>
-    public void Parse(string fileName, bool columnNamesInFirstLine) {
+    public void Parse(string fileName, bool columnNamesInFirstLine, int lineLimit = -1) {
       NumberFormatInfo numberFormat;
       DateTimeFormatInfo dateTimeFormatInfo;
       char separator;
       DetermineFileFormat(fileName, out numberFormat, out dateTimeFormatInfo, out separator);
-      Parse(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine);
+      Parse(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine, lineLimit);
     }
 
@@ -130 +129 @@
     /// <param name="separator">defines the separator</param>
     /// <param name="columnNamesInFirstLine"></param>
-    public void Parse(string fileName, NumberFormatInfo numberFormat, DateTimeFormatInfo dateTimeFormatInfo, char separator, bool columnNamesInFirstLine) {
+    public void Parse(string fileName, NumberFormatInfo numberFormat, DateTimeFormatInfo dateTimeFormatInfo, char separator, bool columnNamesInFirstLine, int lineLimit = -1) {
       using (var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
-        Parse(stream, numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine);
+        Parse(stream, numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine, lineLimit);
       }
     }
@@ -141 +140 @@
     /// <param name="stream">stream which is parsed</param>
     /// <param name="columnNamesInFirstLine"></param>
-    public void Parse(Stream stream, bool columnNamesInFirstLine) {
+    public void Parse(Stream stream, bool columnNamesInFirstLine, int lineLimit = -1) {
       NumberFormatInfo numberFormat = NumberFormatInfo.InvariantInfo;
       DateTimeFormatInfo dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
       char separator = ',';
-      Parse(stream, numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine);
+      Parse(stream, numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine, lineLimit);
     }
 
@@ -156 +155 @@
     /// <param name="separator">defines the separator</param>
     /// <param name="columnNamesInFirstLine"></param>
-    public void Parse(Stream stream, NumberFormatInfo numberFormat, DateTimeFormatInfo dateTimeFormatInfo, char separator, bool columnNamesInFirstLine) {
+    public void Parse(Stream stream, NumberFormatInfo numberFormat, DateTimeFormatInfo dateTimeFormatInfo, char separator, bool columnNamesInFirstLine, int lineLimit = -1) {
       using (StreamReader reader = new StreamReader(stream)) {
         tokenizer = new Tokenizer(reader, numberFormat, dateTimeFormatInfo, separator);
         // parse the file
-        Parse(columnNamesInFirstLine);
+        Parse(columnNamesInFirstLine, lineLimit);
       }
 
@@ -287 +286 @@
     }
 
-
-    //internal class Token {
-    //  public TokenTypeEnum type;
-    //  public string stringValue;
-    //  public double doubleValue;
-    //  public DateTime dateTimeValue;
-    //
-    //  public Token(TokenTypeEnum type, string value) {
-    //    this.type = type;
-    //    stringValue = value;
-    //    dateTimeValue = DateTime.MinValue;
-    //    doubleValue = 0.0;
-    //  }
-    //
-    //  public bool Equals(Token other) {
-    //    throw new NotImplementedException();
-    //  }
-    //
-    //  public override string ToString() {
-    //    return stringValue;
-    //  }
-    //
-    //  public override bool Equals(object obj) {
-    //    return Equals(obj as Token);
-    //  }
-    //
-    //  public override int GetHashCode() {
-    //    throw new NotSupportedException();
-    //  }
-    //}
-
-
     internal class Tokenizer {
       private StreamReader reader;
@@ -344 +311 @@
       }
 
-      // private Token newlineToken;
-      // public Token NewlineToken {
-      //   get { return newlineToken; }
-      //   private set { newlineToken = value; }
-      // }
-      // private Token separatorToken;
-      // public Token SeparatorToken {
-      //   get { return separatorToken; }
-      //   private set { separatorToken = value; }
-      // }
-
       public Tokenizer(StreamReader reader, NumberFormatInfo numberFormatInfo, DateTimeFormatInfo dateTimeFormatInfo, char separator) {
         this.reader = reader;
@@ -360 +316 @@
         this.dateTimeFormatInfo = dateTimeFormatInfo;
         this.separator = separator;
-        //separatorToken = new Token(TokenTypeEnum.Separator, INTERNAL_SEPARATOR);
-        //newlineToken = new Token(TokenTypeEnum.NewLine, Environment.NewLine);
         ReadNextTokens();
       }
@@ -458 +412 @@
 
     #region parsing
-    private void Parse(bool columnNamesInFirstLine) {
+    private void Parse(bool columnNamesInFirstLine, int lineLimit = -1) { // lineLimit = -1 means no limit
       if (columnNamesInFirstLine) {
         ParseVariableNames();
@@ -466 +420 @@
             "", tokenizer.CurrentLineNumber);
       }
-      ParseValues();
+      ParseValues(lineLimit);
       if (rowValues.Count == 0) Error("Couldn't parse data values. Probably because of incorrect number format (the parser expects english number format with a '.' as decimal separator).", "", tokenizer.CurrentLineNumber);
     }
 
-    private void ParseValues() {
-      while (tokenizer.HasNext()) {
+    private void ParseValues(int lineLimit = -1) {
+      int nLinesParsed = 0;
+      while (tokenizer.HasNext() && (lineLimit < 0 || nLinesParsed < lineLimit)) {
         if (tokenizer.PeekType() == TokenTypeEnum.NewLine) {
           tokenizer.Skip();
+          nLinesParsed++;
         } else {
           List<object> row = new List<object>();
@@ -483 +439 @@
           }
           ExpectType(TokenTypeEnum.NewLine);
+          nLinesParsed++;
           // all rows have to have the same number of values            
           // the first row defines how many samples are needed