[7849] | 1 | #region License Information
|
---|
| 2 | /* HeuristicLab
|
---|
[16140] | 3 | * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
|
---|
[7849] | 4 | *
|
---|
| 5 | * This file is part of HeuristicLab.
|
---|
| 6 | *
|
---|
| 7 | * HeuristicLab is free software: you can redistribute it and/or modify
|
---|
| 8 | * it under the terms of the GNU General Public License as published by
|
---|
| 9 | * the Free Software Foundation, either version 3 of the License, or
|
---|
| 10 | * (at your option) any later version.
|
---|
| 11 | *
|
---|
| 12 | * HeuristicLab is distributed in the hope that it will be useful,
|
---|
| 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
---|
| 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
---|
| 15 | * GNU General Public License for more details.
|
---|
| 16 | *
|
---|
| 17 | * You should have received a copy of the GNU General Public License
|
---|
| 18 | * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
|
---|
| 19 | */
|
---|
| 20 | #endregion
|
---|
| 21 |
|
---|
| 22 |
|
---|
| 23 | using System;
|
---|
| 24 | using System.Collections;
|
---|
| 25 | using System.Collections.Generic;
|
---|
[13526] | 26 | using System.Diagnostics.Contracts;
|
---|
[7849] | 27 | using System.Globalization;
|
---|
| 28 | using System.IO;
|
---|
| 29 | using System.Linq;
|
---|
[13440] | 30 | using System.Text;
|
---|
[7849] | 31 |
|
---|
| 32 | namespace HeuristicLab.Problems.Instances.DataAnalysis {
|
---|
[13414] | 33 | public class TableFileParser : Progress<long> { // reports the number of bytes read
|
---|
[8564] | 34 | private const int BUFFER_SIZE = 65536;
|
---|
[9652] | 35 | // char used to symbolize whitespaces (no missing values can be handled with whitespaces)
|
---|
| 36 | private const char WHITESPACECHAR = (char)0;
|
---|
| 37 | private static readonly char[] POSSIBLE_SEPARATORS = new char[] { ',', ';', '\t', WHITESPACECHAR };
|
---|
[7849] | 38 | private Tokenizer tokenizer;
|
---|
[13440] | 39 | private int estimatedNumberOfLines = 200; // initial capacity for columns, will be set automatically when data is read from a file
|
---|
[7849] | 40 |
|
---|
[13584] | 41 |
|
---|
| 42 | private Encoding encoding = Encoding.Default;
|
---|
| 43 |
|
---|
| 44 | public Encoding Encoding {
|
---|
| 45 | get { return encoding; }
|
---|
| 46 | set {
|
---|
| 47 | if (value == null) throw new ArgumentNullException("Encoding");
|
---|
| 48 | encoding = value;
|
---|
| 49 | }
|
---|
| 50 | }
|
---|
| 51 |
|
---|
| 52 |
|
---|
[7849] | 53 | private int rows;
|
---|
| 54 | public int Rows {
|
---|
| 55 | get { return rows; }
|
---|
| 56 | set { rows = value; }
|
---|
| 57 | }
|
---|
| 58 |
|
---|
| 59 | private int columns;
|
---|
| 60 | public int Columns {
|
---|
| 61 | get { return columns; }
|
---|
| 62 | set { columns = value; }
|
---|
| 63 | }
|
---|
| 64 |
|
---|
| 65 | private List<IList> values;
|
---|
| 66 | public List<IList> Values {
|
---|
| 67 | get {
|
---|
| 68 | return values;
|
---|
| 69 | }
|
---|
| 70 | }
|
---|
| 71 |
|
---|
| 72 | private List<string> variableNames;
|
---|
| 73 | public IEnumerable<string> VariableNames {
|
---|
| 74 | get {
|
---|
| 75 | if (variableNames.Count > 0) return variableNames;
|
---|
| 76 | else {
|
---|
| 77 | string[] names = new string[columns];
|
---|
| 78 | for (int i = 0; i < names.Length; i++) {
|
---|
| 79 | names[i] = "X" + i.ToString("000");
|
---|
| 80 | }
|
---|
| 81 | return names;
|
---|
| 82 | }
|
---|
| 83 | }
|
---|
| 84 | }
|
---|
| 85 |
|
---|
| 86 | public TableFileParser() {
|
---|
| 87 | variableNames = new List<string>();
|
---|
| 88 | }
|
---|
| 89 |
|
---|
[9608] | 90 | public bool AreColumnNamesInFirstLine(string fileName) {
|
---|
| 91 | NumberFormatInfo numberFormat;
|
---|
| 92 | DateTimeFormatInfo dateTimeFormatInfo;
|
---|
| 93 | char separator;
|
---|
| 94 | DetermineFileFormat(fileName, out numberFormat, out dateTimeFormatInfo, out separator);
|
---|
| 95 | using (var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
|
---|
| 96 | return AreColumnNamesInFirstLine(stream, numberFormat, dateTimeFormatInfo, separator);
|
---|
| 97 | }
|
---|
| 98 | }
|
---|
| 99 |
|
---|
| 100 | public bool AreColumnNamesInFirstLine(Stream stream) {
|
---|
| 101 | NumberFormatInfo numberFormat = NumberFormatInfo.InvariantInfo;
|
---|
| 102 | DateTimeFormatInfo dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
|
---|
| 103 | char separator = ',';
|
---|
| 104 | return AreColumnNamesInFirstLine(stream, numberFormat, dateTimeFormatInfo, separator);
|
---|
| 105 | }
|
---|
| 106 |
|
---|
| 107 | public bool AreColumnNamesInFirstLine(string fileName, NumberFormatInfo numberFormat,
|
---|
| 108 | DateTimeFormatInfo dateTimeFormatInfo, char separator) {
|
---|
| 109 | using (var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
|
---|
| 110 | return AreColumnNamesInFirstLine(stream, numberFormat, dateTimeFormatInfo, separator);
|
---|
| 111 | }
|
---|
| 112 | }
|
---|
| 113 |
|
---|
| 114 | public bool AreColumnNamesInFirstLine(Stream stream, NumberFormatInfo numberFormat,
|
---|
| 115 | DateTimeFormatInfo dateTimeFormatInfo, char separator) {
|
---|
[13584] | 116 | using (StreamReader reader = new StreamReader(stream, Encoding)) {
|
---|
[9608] | 117 | tokenizer = new Tokenizer(reader, numberFormat, dateTimeFormatInfo, separator);
|
---|
[13440] | 118 | return (tokenizer.PeekType() != TokenTypeEnum.Double);
|
---|
[9608] | 119 | }
|
---|
| 120 | }
|
---|
| 121 |
|
---|
[7851] | 122 | /// <summary>
|
---|
| 123 | /// Parses a file and determines the format first
|
---|
| 124 | /// </summary>
|
---|
| 125 | /// <param name="fileName">file which is parsed</param>
|
---|
[9608] | 126 | /// <param name="columnNamesInFirstLine"></param>
|
---|
[13413] | 127 | public void Parse(string fileName, bool columnNamesInFirstLine, int lineLimit = -1) {
|
---|
[7851] | 128 | NumberFormatInfo numberFormat;
|
---|
| 129 | DateTimeFormatInfo dateTimeFormatInfo;
|
---|
| 130 | char separator;
|
---|
[9608] | 131 | DetermineFileFormat(fileName, out numberFormat, out dateTimeFormatInfo, out separator);
|
---|
[13440] | 132 | EstimateNumberOfLines(fileName);
|
---|
[13413] | 133 | Parse(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine, lineLimit);
|
---|
[7851] | 134 | }
|
---|
| 135 |
|
---|
| 136 | /// <summary>
|
---|
| 137 | /// Parses a file with the given formats
|
---|
| 138 | /// </summary>
|
---|
| 139 | /// <param name="fileName">file which is parsed</param>
|
---|
| 140 | /// <param name="numberFormat">Format of numbers</param>
|
---|
| 141 | /// <param name="dateTimeFormatInfo">Format of datetime</param>
|
---|
| 142 | /// <param name="separator">defines the separator</param>
|
---|
[9608] | 143 | /// <param name="columnNamesInFirstLine"></param>
|
---|
[13413] | 144 | public void Parse(string fileName, NumberFormatInfo numberFormat, DateTimeFormatInfo dateTimeFormatInfo, char separator, bool columnNamesInFirstLine, int lineLimit = -1) {
|
---|
[13440] | 145 | EstimateNumberOfLines(fileName);
|
---|
[9608] | 146 | using (var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
|
---|
[13413] | 147 | Parse(stream, numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine, lineLimit);
|
---|
[9608] | 148 | }
|
---|
[7849] | 149 | }
|
---|
| 150 |
|
---|
[13440] | 151 | // determines the number of newline characters in the first 64KB to guess the number of rows for a file
|
---|
| 152 | private void EstimateNumberOfLines(string fileName) {
|
---|
| 153 | var len = new System.IO.FileInfo(fileName).Length;
|
---|
[13526] | 154 | var buf = new char[1024 * 1024];
|
---|
[13584] | 155 | using (var reader = new StreamReader(fileName, Encoding)) {
|
---|
[13445] | 156 | reader.ReadBlock(buf, 0, buf.Length);
|
---|
| 157 | }
|
---|
[13440] | 158 | int numNewLine = 0;
|
---|
[13442] | 159 | int charsInCurrentLine = 0, charsInFirstLine = 0; // the first line (names) and the last line (incomplete) are not representative
|
---|
| 160 | foreach (var ch in buf) {
|
---|
| 161 | charsInCurrentLine++;
|
---|
| 162 | if (ch == '\n') {
|
---|
| 163 | if (numNewLine == 0) charsInFirstLine = charsInCurrentLine; // store the number of chars in the first line
|
---|
| 164 | charsInCurrentLine = 0;
|
---|
| 165 | numNewLine++;
|
---|
| 166 | }
|
---|
| 167 | }
|
---|
| 168 | if (numNewLine <= 1) {
|
---|
[13440] | 169 | // fail -> keep the default setting
|
---|
| 170 | return;
|
---|
| 171 | } else {
|
---|
[13442] | 172 | double charsPerLineFactor = (buf.Length - charsInFirstLine - charsInCurrentLine) / ((double)numNewLine - 1);
|
---|
[13440] | 173 | double estimatedLines = len / charsPerLineFactor;
|
---|
| 174 | estimatedNumberOfLines = (int)Math.Round(estimatedLines * 1.1); // pessimistic allocation of 110% to make sure that the list is very likely large enough
|
---|
| 175 | }
|
---|
| 176 | }
|
---|
| 177 |
|
---|
[7851] | 178 | /// <summary>
|
---|
| 179 | /// Takes a Stream and parses it with default format. NumberFormatInfo.InvariantInfo, DateTimeFormatInfo.InvariantInfo and separator = ','
|
---|
| 180 | /// </summary>
|
---|
| 181 | /// <param name="stream">stream which is parsed</param>
|
---|
[9608] | 182 | /// <param name="columnNamesInFirstLine"></param>
|
---|
[13413] | 183 | public void Parse(Stream stream, bool columnNamesInFirstLine, int lineLimit = -1) {
|
---|
[7851] | 184 | NumberFormatInfo numberFormat = NumberFormatInfo.InvariantInfo;
|
---|
| 185 | DateTimeFormatInfo dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
|
---|
| 186 | char separator = ',';
|
---|
[13413] | 187 | Parse(stream, numberFormat, dateTimeFormatInfo, separator, columnNamesInFirstLine, lineLimit);
|
---|
[7851] | 188 | }
|
---|
| 189 |
|
---|
| 190 | /// <summary>
|
---|
| 191 | /// Parses a stream with the given formats.
|
---|
| 192 | /// </summary>
|
---|
| 193 | /// <param name="stream">Stream which is parsed</param>
|
---|
| 194 | /// <param name="numberFormat">Format of numbers</param>
|
---|
| 195 | /// <param name="dateTimeFormatInfo">Format of datetime</param>
|
---|
| 196 | /// <param name="separator">defines the separator</param>
|
---|
[9608] | 197 | /// <param name="columnNamesInFirstLine"></param>
|
---|
[13413] | 198 | public void Parse(Stream stream, NumberFormatInfo numberFormat, DateTimeFormatInfo dateTimeFormatInfo, char separator, bool columnNamesInFirstLine, int lineLimit = -1) {
|
---|
[14296] | 199 | if (lineLimit > 0) estimatedNumberOfLines = lineLimit;
|
---|
| 200 |
|
---|
| 201 | using (var reader = new StreamReader(stream)) {
|
---|
[7849] | 202 | tokenizer = new Tokenizer(reader, numberFormat, dateTimeFormatInfo, separator);
|
---|
[14296] | 203 | var strValues = new List<List<string>>();
|
---|
[13440] | 204 | values = new List<IList>();
|
---|
[14296] | 205 | Prepare(columnNamesInFirstLine, strValues);
|
---|
[13447] | 206 |
|
---|
| 207 | int nLinesParsed = 0;
|
---|
| 208 | int colIdx = 0;
|
---|
| 209 | while (tokenizer.HasNext() && (lineLimit < 0 || nLinesParsed < lineLimit)) {
|
---|
| 210 | if (tokenizer.PeekType() == TokenTypeEnum.NewLine) {
|
---|
| 211 | tokenizer.Skip();
|
---|
| 212 |
|
---|
| 213 | // all rows have to have the same number of values
|
---|
[14296] | 214 | // the first row defines how many elements are needed
|
---|
| 215 | if (colIdx > 0 && values.Count != colIdx) {
|
---|
| 216 | // read at least one value in the row (support for skipping empty lines)
|
---|
| 217 | Error("The first row of the dataset has " + values.Count + " columns." + Environment.NewLine +
|
---|
[13447] | 218 | "Line " + tokenizer.CurrentLineNumber + " has " + colIdx + " columns.", "",
|
---|
[14296] | 219 | tokenizer.CurrentLineNumber);
|
---|
[13440] | 220 | }
|
---|
[13447] | 221 | OnReport(tokenizer.BytesRead);
|
---|
| 222 |
|
---|
| 223 | nLinesParsed++;
|
---|
| 224 | colIdx = 0;
|
---|
[13440] | 225 | } else {
|
---|
[13447] | 226 | // read one value
|
---|
[14296] | 227 | TokenTypeEnum type;
|
---|
| 228 | string strVal;
|
---|
| 229 | double dblVal;
|
---|
| 230 | DateTime dateTimeVal;
|
---|
[13447] | 231 | tokenizer.Next(out type, out strVal, out dblVal, out dateTimeVal);
|
---|
| 232 |
|
---|
[14296] | 233 | if (colIdx == values.Count) {
|
---|
| 234 | Error("The first row of the dataset has " + values.Count + " columns." + Environment.NewLine +
|
---|
[13447] | 235 | "Line " + tokenizer.CurrentLineNumber + " has more columns.", "",
|
---|
| 236 | tokenizer.CurrentLineNumber);
|
---|
[13440] | 237 | }
|
---|
[13526] | 238 | if (!IsColumnTypeCompatible(values[colIdx], type)) {
|
---|
[14296] | 239 | values[colIdx] = strValues[colIdx];
|
---|
[13526] | 240 | }
|
---|
[14296] | 241 |
|
---|
[13447] | 242 | // add the value to the column
|
---|
[14296] | 243 | AddValue(type, values[colIdx], strVal, dblVal, dateTimeVal);
|
---|
| 244 | if (!(values[colIdx] is List<string>)) { // optimization: don't store the string values in another list if the column is list<string>
|
---|
| 245 | strValues[colIdx].Add(strVal);
|
---|
| 246 | }
|
---|
| 247 | colIdx++;
|
---|
[13440] | 248 | }
|
---|
[7849] | 249 | }
|
---|
| 250 | }
|
---|
| 251 |
|
---|
[14296] | 252 | if (!values.Any() || values.First().Count == 0)
|
---|
| 253 | Error("Couldn't parse data values. Probably because of incorrect number format " +
|
---|
| 254 | "(the parser expects english number format with a '.' as decimal separator).", "", tokenizer.CurrentLineNumber);
|
---|
| 255 |
|
---|
[13441] | 256 | this.rows = values.First().Count;
|
---|
[13447] | 257 | this.columns = values.Count;
|
---|
[13441] | 258 |
|
---|
[16140] | 259 | // replace lists with undefined type (object) with double-lists
|
---|
| 260 | for (int i = 0; i < values.Count; i++) {
|
---|
| 261 | if (values[i] is List<object>) {
|
---|
| 262 | values[i] = Enumerable.Repeat(double.NaN, rows).ToList();
|
---|
| 263 | }
|
---|
| 264 | }
|
---|
| 265 |
|
---|
[13440] | 266 | // after everything has been parsed make sure the lists are as compact as possible
|
---|
| 267 | foreach (var l in values) {
|
---|
| 268 | var dblList = l as List<double>;
|
---|
| 269 | var byteList = l as List<byte>;
|
---|
| 270 | var dateList = l as List<DateTime>;
|
---|
| 271 | var stringList = l as List<string>;
|
---|
| 272 | var objList = l as List<object>;
|
---|
| 273 | if (dblList != null) dblList.TrimExcess();
|
---|
| 274 | if (byteList != null) byteList.TrimExcess();
|
---|
| 275 | if (dateList != null) dateList.TrimExcess();
|
---|
| 276 | if (stringList != null) stringList.TrimExcess();
|
---|
| 277 | if (objList != null) objList.TrimExcess();
|
---|
[7849] | 278 | }
|
---|
[13442] | 279 |
|
---|
| 280 | // for large files we created a lot of memory pressure, cannot hurt to run GC.Collect here (TableFileParser is called seldomly on user interaction)
|
---|
| 281 | GC.Collect(2, GCCollectionMode.Forced);
|
---|
[7849] | 282 | }
|
---|
| 283 |
|
---|
[14296] | 284 | private void Prepare(bool columnNamesInFirstLine, List<List<string>> strValues) {
|
---|
| 285 | if (columnNamesInFirstLine) {
|
---|
| 286 | ParseVariableNames();
|
---|
| 287 | if (!tokenizer.HasNext())
|
---|
| 288 | Error(
|
---|
| 289 | "Couldn't parse data values. Probably because of incorrect number format (the parser expects english number format with a '.' as decimal separator).",
|
---|
| 290 | "", tokenizer.CurrentLineNumber);
|
---|
| 291 | }
|
---|
| 292 | // read first line to determine types and allocate specific lists
|
---|
| 293 | // read values... start in first row
|
---|
| 294 | int colIdx = 0;
|
---|
| 295 | while (tokenizer.PeekType() != TokenTypeEnum.NewLine) {
|
---|
| 296 | // read one value
|
---|
| 297 | TokenTypeEnum type; string strVal; double dblVal; DateTime dateTimeVal;
|
---|
| 298 | tokenizer.Next(out type, out strVal, out dblVal, out dateTimeVal);
|
---|
| 299 |
|
---|
| 300 | // initialize column
|
---|
| 301 | values.Add(CreateList(type, estimatedNumberOfLines));
|
---|
| 302 | if (type == TokenTypeEnum.String)
|
---|
| 303 | strValues.Add(new List<string>(0)); // optimization: don't store the string values in another list if the column is list<string>
|
---|
| 304 | else
|
---|
| 305 | strValues.Add(new List<string>(estimatedNumberOfLines));
|
---|
| 306 |
|
---|
| 307 | AddValue(type, values[colIdx], strVal, dblVal, dateTimeVal);
|
---|
| 308 | if (type != TokenTypeEnum.String)
|
---|
| 309 | strValues[colIdx].Add(strVal);
|
---|
| 310 | colIdx++;
|
---|
| 311 | }
|
---|
| 312 | tokenizer.Skip(); // skip newline
|
---|
| 313 | }
|
---|
| 314 |
|
---|
[13447] | 315 | #region type-dependent dispatch
|
---|
[13526] | 316 | private bool IsColumnTypeCompatible(IList list, TokenTypeEnum tokenType) {
|
---|
[16140] | 317 | return (list is List<object>) || // unknown lists are compatible to everything (potential conversion)
|
---|
| 318 | (list is List<string>) || // all tokens can be added to a string list
|
---|
[13526] | 319 | (tokenType == TokenTypeEnum.Missing) || // empty entries are allowed in all columns
|
---|
| 320 | (tokenType == TokenTypeEnum.Double && list is List<double>) ||
|
---|
| 321 | (tokenType == TokenTypeEnum.DateTime && list is List<DateTime>);
|
---|
| 322 | }
|
---|
| 323 |
|
---|
| 324 | // all columns are converted to string columns when we find an non-empty value that has incorrect type
|
---|
| 325 | private IList ConvertToStringColumn(IList list) {
|
---|
| 326 | var dblL = list as List<double>;
|
---|
| 327 | if (dblL != null) {
|
---|
| 328 | var l = new List<string>(dblL.Capacity);
|
---|
| 329 | l.AddRange(dblL.Select(dbl => dbl.ToString()));
|
---|
| 330 | return l;
|
---|
[13447] | 331 | }
|
---|
[13526] | 332 |
|
---|
| 333 | var dtL = list as List<DateTime>;
|
---|
| 334 | if (dtL != null) {
|
---|
| 335 | var l = new List<string>(dtL.Capacity);
|
---|
| 336 | l.AddRange(dtL.Select(dbl => dbl.ToString()));
|
---|
| 337 | return l;
|
---|
| 338 | }
|
---|
| 339 |
|
---|
| 340 | if (list is List<string>) return list;
|
---|
| 341 |
|
---|
| 342 | throw new InvalidProgramException(string.Format("Cannot convert column of type {0} to string column", list.GetType()));
|
---|
[13447] | 343 | }
|
---|
| 344 |
|
---|
[13526] | 345 | private void AddValue(TokenTypeEnum type, IList list, string strVal, double dblVal, DateTime dateTimeVal) {
|
---|
[16140] | 346 | // Add value if list has a defined type
|
---|
[13447] | 347 | var dblList = list as List<double>;
|
---|
[13526] | 348 | if (dblList != null) {
|
---|
| 349 | AddValue(type, dblList, dblVal);
|
---|
| 350 | return;
|
---|
[13447] | 351 | }
|
---|
| 352 | var strList = list as List<string>;
|
---|
[13526] | 353 | if (strList != null) {
|
---|
| 354 | AddValue(type, strList, strVal);
|
---|
| 355 | return;
|
---|
[13447] | 356 | }
|
---|
[13526] | 357 | var dtList = list as List<DateTime>;
|
---|
| 358 | if (dtList != null) {
|
---|
| 359 | AddValue(type, dtList, dateTimeVal);
|
---|
| 360 | return;
|
---|
| 361 | }
|
---|
| 362 |
|
---|
[16140] | 363 | // Undefined list-type
|
---|
| 364 | if (type == TokenTypeEnum.Missing) {
|
---|
| 365 | // add null to track number of missing values
|
---|
| 366 | list.Add(null);
|
---|
| 367 | } else { // first non-missing value for undefined list-type
|
---|
| 368 | var newList = ConvertList(type, list, estimatedNumberOfLines);
|
---|
| 369 | // replace list
|
---|
| 370 | var idx = values.IndexOf(list);
|
---|
| 371 | values[idx] = newList;
|
---|
| 372 | // recursively call AddValue
|
---|
| 373 | AddValue(type, newList, strVal, dblVal, dateTimeVal);
|
---|
| 374 | }
|
---|
[13447] | 375 | }
|
---|
| 376 |
|
---|
[16140] | 377 | private static void AddValue(TokenTypeEnum type, List<double> list, double dblVal) {
|
---|
[13526] | 378 | Contract.Assert(type == TokenTypeEnum.Missing || type == TokenTypeEnum.Double);
|
---|
| 379 | list.Add(type == TokenTypeEnum.Missing ? double.NaN : dblVal);
|
---|
[13447] | 380 | }
|
---|
| 381 |
|
---|
[16140] | 382 | private static void AddValue(TokenTypeEnum type, List<string> list, string strVal) {
|
---|
[13526] | 383 | // assumes that strVal is always set to the original token read from the input file
|
---|
| 384 | list.Add(type == TokenTypeEnum.Missing ? string.Empty : strVal);
|
---|
| 385 | }
|
---|
| 386 |
|
---|
[16140] | 387 | private static void AddValue(TokenTypeEnum type, List<DateTime> list, DateTime dtVal) {
|
---|
[13526] | 388 | Contract.Assert(type == TokenTypeEnum.Missing || type == TokenTypeEnum.DateTime);
|
---|
| 389 | list.Add(type == TokenTypeEnum.Missing ? DateTime.MinValue : dtVal);
|
---|
| 390 | }
|
---|
| 391 |
|
---|
[16140] | 392 | private static IList CreateList(TokenTypeEnum type, int estimatedNumberOfLines) {
|
---|
[13447] | 393 | switch (type) {
|
---|
| 394 | case TokenTypeEnum.String:
|
---|
| 395 | return new List<string>(estimatedNumberOfLines);
|
---|
| 396 | case TokenTypeEnum.Double:
|
---|
| 397 | return new List<double>(estimatedNumberOfLines);
|
---|
| 398 | case TokenTypeEnum.DateTime:
|
---|
| 399 | return new List<DateTime>(estimatedNumberOfLines);
|
---|
[16140] | 400 | case TokenTypeEnum.Missing: // List<object> represent list of unknown type
|
---|
| 401 | return new List<object>(estimatedNumberOfLines);
|
---|
[13447] | 402 | default:
|
---|
| 403 | throw new InvalidOperationException();
|
---|
| 404 | }
|
---|
| 405 | }
|
---|
[16140] | 406 |
|
---|
| 407 | private static IList ConvertList(TokenTypeEnum type, IList list, int estimatedNumberOfLines) {
|
---|
| 408 | var newList = CreateList(type, estimatedNumberOfLines);
|
---|
| 409 | object missingValue = GetMissingValue(type);
|
---|
| 410 | for (int i = 0; i < list.Count; i++)
|
---|
| 411 | newList.Add(missingValue);
|
---|
| 412 | return newList;
|
---|
| 413 | }
|
---|
| 414 | private static object GetMissingValue(TokenTypeEnum type) {
|
---|
| 415 | switch (type) {
|
---|
| 416 | case TokenTypeEnum.String: return string.Empty;
|
---|
| 417 | case TokenTypeEnum.Double: return double.NaN;
|
---|
| 418 | case TokenTypeEnum.DateTime: return DateTime.MinValue;
|
---|
| 419 | default: throw new ArgumentOutOfRangeException("type", type, "No missing value defined");
|
---|
| 420 | }
|
---|
| 421 | }
|
---|
[13447] | 422 | #endregion
|
---|
| 423 |
|
---|
[7849] | 424 | public static void DetermineFileFormat(string path, out NumberFormatInfo numberFormat, out DateTimeFormatInfo dateTimeFormatInfo, out char separator) {
|
---|
[8885] | 425 | DetermineFileFormat(new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), out numberFormat, out dateTimeFormatInfo, out separator);
|
---|
[7849] | 426 | }
|
---|
| 427 |
|
---|
| 428 | public static void DetermineFileFormat(Stream stream, out NumberFormatInfo numberFormat, out DateTimeFormatInfo dateTimeFormatInfo, out char separator) {
|
---|
| 429 | using (StreamReader reader = new StreamReader(stream)) {
|
---|
| 430 | // skip first line
|
---|
| 431 | reader.ReadLine();
|
---|
| 432 | // read a block
|
---|
| 433 | char[] buffer = new char[BUFFER_SIZE];
|
---|
| 434 | int charsRead = reader.ReadBlock(buffer, 0, BUFFER_SIZE);
|
---|
| 435 | // count frequency of special characters
|
---|
| 436 | Dictionary<char, int> charCounts = buffer.Take(charsRead)
|
---|
| 437 | .GroupBy(c => c)
|
---|
| 438 | .ToDictionary(g => g.Key, g => g.Count());
|
---|
| 439 |
|
---|
| 440 | // depending on the characters occuring in the block
|
---|
| 441 | // we distinghish a number of different cases based on the the following rules:
|
---|
| 442 | // many points => it must be English number format, the other frequently occuring char is the separator
|
---|
| 443 | // no points but many commas => this is the problematic case. Either German format (real numbers) or English format (only integer numbers) with ',' as separator
|
---|
| 444 | // => check the line in more detail:
|
---|
| 445 | // English: 0, 0, 0, 0
|
---|
| 446 | // German: 0,0 0,0 0,0 ...
|
---|
| 447 | // => if commas are followed by space => English format
|
---|
| 448 | // no points no commas => English format (only integer numbers) use the other frequently occuring char as separator
|
---|
| 449 | // in all cases only treat ' ' as separator if no other separator is possible (spaces can also occur additionally to separators)
|
---|
| 450 | if (OccurrencesOf(charCounts, '.') > 10) {
|
---|
| 451 | numberFormat = NumberFormatInfo.InvariantInfo;
|
---|
| 452 | dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
|
---|
| 453 | separator = POSSIBLE_SEPARATORS
|
---|
| 454 | .Where(c => OccurrencesOf(charCounts, c) > 10)
|
---|
| 455 | .OrderBy(c => -OccurrencesOf(charCounts, c))
|
---|
| 456 | .DefaultIfEmpty(' ')
|
---|
| 457 | .First();
|
---|
| 458 | } else if (OccurrencesOf(charCounts, ',') > 10) {
|
---|
| 459 | // no points and many commas
|
---|
| 460 | // count the number of tokens (chains of only digits and commas) that contain multiple comma characters
|
---|
| 461 | int tokensWithMultipleCommas = 0;
|
---|
| 462 | for (int i = 0; i < charsRead; i++) {
|
---|
| 463 | int nCommas = 0;
|
---|
| 464 | while (i < charsRead && (buffer[i] == ',' || Char.IsDigit(buffer[i]))) {
|
---|
| 465 | if (buffer[i] == ',') nCommas++;
|
---|
| 466 | i++;
|
---|
| 467 | }
|
---|
| 468 | if (nCommas > 2) tokensWithMultipleCommas++;
|
---|
| 469 | }
|
---|
| 470 | if (tokensWithMultipleCommas > 1) {
|
---|
| 471 | // English format (only integer values) with ',' as separator
|
---|
| 472 | numberFormat = NumberFormatInfo.InvariantInfo;
|
---|
| 473 | dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
|
---|
| 474 | separator = ',';
|
---|
| 475 | } else {
|
---|
[13526] | 476 | char[] disallowedSeparators = new char[] { ',' }; // n. def. contains a space so ' ' should be disallowed to, however existing unit tests would fail
|
---|
[7849] | 477 | // German format (real values)
|
---|
| 478 | numberFormat = NumberFormatInfo.GetInstance(new CultureInfo("de-DE"));
|
---|
| 479 | dateTimeFormatInfo = DateTimeFormatInfo.GetInstance(new CultureInfo("de-DE"));
|
---|
| 480 | separator = POSSIBLE_SEPARATORS
|
---|
| 481 | .Except(disallowedSeparators)
|
---|
| 482 | .Where(c => OccurrencesOf(charCounts, c) > 10)
|
---|
| 483 | .OrderBy(c => -OccurrencesOf(charCounts, c))
|
---|
[13584] | 484 | .DefaultIfEmpty(' ')
|
---|
[7849] | 485 | .First();
|
---|
| 486 | }
|
---|
| 487 | } else {
|
---|
| 488 | // no points and no commas => English format
|
---|
| 489 | numberFormat = NumberFormatInfo.InvariantInfo;
|
---|
| 490 | dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
|
---|
| 491 | separator = POSSIBLE_SEPARATORS
|
---|
| 492 | .Where(c => OccurrencesOf(charCounts, c) > 10)
|
---|
| 493 | .OrderBy(c => -OccurrencesOf(charCounts, c))
|
---|
| 494 | .DefaultIfEmpty(' ')
|
---|
| 495 | .First();
|
---|
| 496 | }
|
---|
| 497 | }
|
---|
| 498 | }
|
---|
| 499 |
|
---|
| 500 | private static int OccurrencesOf(Dictionary<char, int> charCounts, char c) {
|
---|
| 501 | return charCounts.ContainsKey(c) ? charCounts[c] : 0;
|
---|
| 502 | }
|
---|
| 503 |
|
---|
| 504 | #region tokenizer
|
---|
[13447] | 505 | // the tokenizer reads full lines and returns separated tokens in the line as well as a terminating end-of-line character
|
---|
[7849] | 506 | internal enum TokenTypeEnum {
|
---|
[13526] | 507 | NewLine, String, Double, DateTime, Missing
|
---|
[7849] | 508 | }
|
---|
| 509 |
|
---|
| 510 | internal class Tokenizer {
|
---|
| 511 | private StreamReader reader;
|
---|
[13411] | 512 | // we assume that a buffer of 1024 tokens for a line is sufficient most of the time (the buffer is increased below if necessary)
|
---|
| 513 | private TokenTypeEnum[] tokenTypes = new TokenTypeEnum[1024];
|
---|
| 514 | private string[] stringVals = new string[1024];
|
---|
| 515 | private double[] doubleVals = new double[1024];
|
---|
| 516 | private DateTime[] dateTimeVals = new DateTime[1024];
|
---|
| 517 | private int tokenPos;
|
---|
| 518 | private int numTokens;
|
---|
[7849] | 519 | private NumberFormatInfo numberFormatInfo;
|
---|
| 520 | private DateTimeFormatInfo dateTimeFormatInfo;
|
---|
| 521 | private char separator;
|
---|
| 522 |
|
---|
[13447] | 523 | // arrays for string.Split()
|
---|
| 524 | private readonly char[] whiteSpaceSeparators = new char[0]; // string split uses separators as default
|
---|
| 525 | private readonly char[] separators;
|
---|
| 526 |
|
---|
[7849] | 527 | private int currentLineNumber = 0;
|
---|
| 528 | public int CurrentLineNumber {
|
---|
| 529 | get { return currentLineNumber; }
|
---|
| 530 | private set { currentLineNumber = value; }
|
---|
| 531 | }
|
---|
| 532 | private string currentLine;
|
---|
| 533 | public string CurrentLine {
|
---|
| 534 | get { return currentLine; }
|
---|
| 535 | private set { currentLine = value; }
|
---|
| 536 | }
|
---|
[13414] | 537 | public long BytesRead {
|
---|
| 538 | get;
|
---|
| 539 | private set;
|
---|
| 540 | }
|
---|
[7849] | 541 |
|
---|
| 542 | public Tokenizer(StreamReader reader, NumberFormatInfo numberFormatInfo, DateTimeFormatInfo dateTimeFormatInfo, char separator) {
|
---|
| 543 | this.reader = reader;
|
---|
| 544 | this.numberFormatInfo = numberFormatInfo;
|
---|
| 545 | this.dateTimeFormatInfo = dateTimeFormatInfo;
|
---|
| 546 | this.separator = separator;
|
---|
[13447] | 547 | this.separators = new char[] { separator };
|
---|
[7849] | 548 | ReadNextTokens();
|
---|
| 549 | }
|
---|
| 550 |
|
---|
[13447] | 551 | public bool HasNext() {
|
---|
| 552 | return numTokens > tokenPos || !reader.EndOfStream;
|
---|
| 553 | }
|
---|
| 554 |
|
---|
| 555 | public TokenTypeEnum PeekType() {
|
---|
| 556 | return tokenTypes[tokenPos];
|
---|
| 557 | }
|
---|
| 558 |
|
---|
| 559 | public void Skip() {
|
---|
| 560 | // simply skips one token without returning the result values
|
---|
| 561 | tokenPos++;
|
---|
| 562 | if (numTokens == tokenPos) {
|
---|
| 563 | ReadNextTokens();
|
---|
| 564 | }
|
---|
| 565 | }
|
---|
| 566 |
|
---|
| 567 | public void Next(out TokenTypeEnum type, out string strVal, out double dblVal, out DateTime dateTimeVal) {
|
---|
| 568 | type = tokenTypes[tokenPos];
|
---|
| 569 | strVal = stringVals[tokenPos];
|
---|
| 570 | dblVal = doubleVals[tokenPos];
|
---|
| 571 | dateTimeVal = dateTimeVals[tokenPos];
|
---|
| 572 | Skip();
|
---|
| 573 | }
|
---|
| 574 |
|
---|
[7849] | 575 | private void ReadNextTokens() {
|
---|
| 576 | if (!reader.EndOfStream) {
|
---|
| 577 | CurrentLine = reader.ReadLine();
|
---|
[13447] | 578 | CurrentLineNumber++;
|
---|
[13925] | 579 | if (reader.BaseStream.CanSeek) {
|
---|
[13414] | 580 | BytesRead = reader.BaseStream.Position;
|
---|
[13925] | 581 | } else {
|
---|
[13414] | 582 | BytesRead += CurrentLine.Length + 2; // guess
|
---|
[13584] | 583 | }
|
---|
[13411] | 584 | int i = 0;
|
---|
[13447] | 585 | if (!string.IsNullOrWhiteSpace(CurrentLine)) {
|
---|
| 586 | foreach (var tok in Split(CurrentLine)) {
|
---|
| 587 | TokenTypeEnum type;
|
---|
[13411] | 588 | double doubleVal;
|
---|
| 589 | DateTime dateTimeValue;
|
---|
[13447] | 590 | type = TokenTypeEnum.String; // default
|
---|
| 591 | stringVals[i] = tok.Trim();
|
---|
| 592 | if (double.TryParse(tok, NumberStyles.Float, numberFormatInfo, out doubleVal)) {
|
---|
[13411] | 593 | type = TokenTypeEnum.Double;
|
---|
| 594 | doubleVals[i] = doubleVal;
|
---|
[14296] | 595 | } else if (DateTime.TryParse(tok, dateTimeFormatInfo, DateTimeStyles.NoCurrentDateDefault, out dateTimeValue)
|
---|
[14408] | 596 | && (dateTimeValue.Year > 1 || dateTimeValue.Month > 1 || dateTimeValue.Day > 1)// if no date is given it is returned as 1.1.0001 -> don't allow this
|
---|
[14296] | 597 | ) {
|
---|
[13411] | 598 | type = TokenTypeEnum.DateTime;
|
---|
| 599 | dateTimeVals[i] = dateTimeValue;
|
---|
[13526] | 600 | } else if (string.IsNullOrWhiteSpace(tok)) {
|
---|
| 601 | type = TokenTypeEnum.Missing;
|
---|
[13411] | 602 | }
|
---|
[7849] | 603 |
|
---|
[13447] | 604 | // couldn't parse the token as an int or float number or datetime value so return a string token
|
---|
[13411] | 605 |
|
---|
| 606 | tokenTypes[i] = type;
|
---|
| 607 | i++;
|
---|
| 608 |
|
---|
| 609 | if (i >= tokenTypes.Length) {
|
---|
| 610 | // increase buffer size if necessary
|
---|
| 611 | IncreaseCapacity(ref tokenTypes);
|
---|
| 612 | IncreaseCapacity(ref doubleVals);
|
---|
| 613 | IncreaseCapacity(ref stringVals);
|
---|
| 614 | IncreaseCapacity(ref dateTimeVals);
|
---|
| 615 | }
|
---|
| 616 | }
|
---|
| 617 | }
|
---|
| 618 | tokenTypes[i] = TokenTypeEnum.NewLine;
|
---|
| 619 | numTokens = i + 1;
|
---|
| 620 | tokenPos = 0;
|
---|
[7849] | 621 | }
|
---|
| 622 | }
|
---|
| 623 |
|
---|
[13447] | 624 | private IEnumerable<string> Split(string line) {
|
---|
| 625 | return separator == WHITESPACECHAR ?
|
---|
| 626 | line.Split(whiteSpaceSeparators, StringSplitOptions.RemoveEmptyEntries) :
|
---|
| 627 | line.Split(separators);
|
---|
| 628 | }
|
---|
| 629 |
|
---|
[13411] | 630 | private static void IncreaseCapacity<T>(ref T[] arr) {
|
---|
| 631 | int n = (int)Math.Floor(arr.Length * 1.7); // guess
|
---|
| 632 | T[] arr2 = new T[n];
|
---|
| 633 | Array.Copy(arr, arr2, arr.Length);
|
---|
| 634 | arr = arr2;
|
---|
| 635 | }
|
---|
[7849] | 636 | }
|
---|
| 637 | #endregion
|
---|
| 638 |
|
---|
| 639 | #region parsing
|
---|
| 640 |
|
---|
| 641 | private void ParseVariableNames() {
|
---|
| 642 | // the first line must contain variable names
|
---|
[13411] | 643 | List<string> varNames = new List<string>();
|
---|
| 644 |
|
---|
| 645 | TokenTypeEnum type;
|
---|
| 646 | string strVal;
|
---|
| 647 | double dblVal;
|
---|
| 648 | DateTime dateTimeVal;
|
---|
| 649 |
|
---|
| 650 | tokenizer.Next(out type, out strVal, out dblVal, out dateTimeVal);
|
---|
| 651 |
|
---|
| 652 | // the first token must be a variable name
|
---|
| 653 | if (type != TokenTypeEnum.String)
|
---|
| 654 | throw new ArgumentException("Error: Expected " + TokenTypeEnum.String + " got " + type);
|
---|
| 655 | varNames.Add(strVal);
|
---|
| 656 |
|
---|
[13447] | 657 | while (tokenizer.HasNext() && tokenizer.PeekType() != TokenTypeEnum.NewLine) {
|
---|
[13411] | 658 | tokenizer.Next(out type, out strVal, out dblVal, out dateTimeVal);
|
---|
| 659 | varNames.Add(strVal);
|
---|
[7849] | 660 | }
|
---|
[13411] | 661 | ExpectType(TokenTypeEnum.NewLine);
|
---|
| 662 |
|
---|
| 663 | variableNames = varNames;
|
---|
[7849] | 664 | }
|
---|
| 665 |
|
---|
[13411] | 666 | private void ExpectType(TokenTypeEnum expectedToken) {
|
---|
| 667 | if (tokenizer.PeekType() != expectedToken)
|
---|
| 668 | throw new ArgumentException("Error: Expected " + expectedToken + " got " + tokenizer.PeekType());
|
---|
| 669 | tokenizer.Skip();
|
---|
[7849] | 670 | }
|
---|
| 671 |
|
---|
| 672 | private void Error(string message, string token, int lineNumber) {
|
---|
[14285] | 673 | throw new IOException(string.Format("Error while parsing. {0} (token: {1} lineNumber: {2}).", message, token, lineNumber));
|
---|
[7849] | 674 | }
|
---|
| 675 | #endregion
|
---|
| 676 | }
|
---|
| 677 | }
|
---|