Changeset 13442 for trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis

Timestamp:

12/07/15 18:44:02 (9 years ago)

Author:

gkronber

Message:

#2071 better estimation of number of rows and explicit call of GC.Collect

File:

• trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/TableFileParser.cs (modified) (3 diffs)

trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/TableFileParser.cs

@@ -27 +27 @@
 using System.IO;
 using System.Linq;
+using System.Runtime;
 using System.Runtime.Serialization;
 using System.Text;
@@ -144 +145 @@
       reader.ReadBlock(buf, 0, buf.Length);
       int numNewLine = 0;
-      foreach (var ch in buf) if (ch == '\n') numNewLine++;
-      if (numNewLine == 0) {
+      int charsInCurrentLine = 0, charsInFirstLine = 0; // the first line (names) and the last line (incomplete) are not representative
+      foreach (var ch in buf) {
+        charsInCurrentLine++;
+        if (ch == '\n') {
+          if (numNewLine == 0) charsInFirstLine = charsInCurrentLine; // store the number of chars in the first line
+          charsInCurrentLine = 0;
+          numNewLine++;
+        }
+      }
+      if (numNewLine <= 1) {
         // fail -> keep the default setting
         return;
       } else {
-        double charsPerLineFactor = buf.Length / (double)numNewLine;
+        double charsPerLineFactor = (buf.Length - charsInFirstLine - charsInCurrentLine) / ((double)numNewLine - 1);
         double estimatedLines = len / charsPerLineFactor;
         estimatedNumberOfLines = (int)Math.Round(estimatedLines * 1.1); // pessimistic allocation of 110% to make sure that the list is very likely large enough
@@ -242 +251 @@
         if (objList != null) objList.TrimExcess();
       }
+
+      // for large files we created a lot of memory pressure, cannot hurt to run GC.Collect here (TableFileParser is called seldomly on user interaction)
+      GC.Collect(2, GCCollectionMode.Forced);
     }