Changeset 13442


Ignore:
Timestamp:
12/07/15 18:44:02 (4 years ago)
Author:
gkronber
Message:

#2072 better estimation of number of rows and explicit call of GC.Collect

File:
1 edited

Legend:

Unmodified
Added
Removed
  • trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/TableFileParser.cs

    r13441 r13442  
    2727using System.IO;
    2828using System.Linq;
     29using System.Runtime;
    2930using System.Runtime.Serialization;
    3031using System.Text;
     
    144145      reader.ReadBlock(buf, 0, buf.Length);
    145146      int numNewLine = 0;
    146       foreach (var ch in buf) if (ch == '\n') numNewLine++;
    147       if (numNewLine == 0) {
     147      int charsInCurrentLine = 0, charsInFirstLine = 0; // the first line (names) and the last line (incomplete) are not representative
     148      foreach (var ch in buf) {
     149        charsInCurrentLine++;
     150        if (ch == '\n') {
     151          if (numNewLine == 0) charsInFirstLine = charsInCurrentLine; // store the number of chars in the first line
     152          charsInCurrentLine = 0;
     153          numNewLine++;
     154        }
     155      }
     156      if (numNewLine <= 1) {
    148157        // fail -> keep the default setting
    149158        return;
    150159      } else {
    151         double charsPerLineFactor = buf.Length / (double)numNewLine;
     160        double charsPerLineFactor = (buf.Length - charsInFirstLine - charsInCurrentLine) / ((double)numNewLine - 1);
    152161        double estimatedLines = len / charsPerLineFactor;
    153162        estimatedNumberOfLines = (int)Math.Round(estimatedLines * 1.1); // pessimistic allocation of 110% to make sure that the list is very likely large enough
     
    242251        if (objList != null) objList.TrimExcess();
    243252      }
     253
     254      // for large files we created a lot of memory pressure, cannot hurt to run GC.Collect here (TableFileParser is called seldomly on user interaction)
     255      GC.Collect(2, GCCollectionMode.Forced);
    244256    }
    245257
Note: See TracChangeset for help on using the changeset viewer.