Free cookie consent management tool by TermsFeed Policy Generator

source: branches/3040_VectorBasedGP/HeuristicLab.Problems.Instances.DataAnalysis/3.3/DataAnalysisInstanceProvider.cs @ 17418

Last change on this file since 17418 was 17414, checked in by pfleck, 5 years ago

#3040 Started adding UCI time series regression benchmarks.
Adapted parser (extracted format options & added parsing for double vectors).

File size: 4.4 KB
RevLine 
[8598]1#region License Information
2/* HeuristicLab
[17180]3 * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
[8598]4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections;
24using System.Collections.Generic;
[13414]25using System.ComponentModel;
[8877]26using System.Globalization;
27using System.IO;
[8598]28using System.Linq;
[8877]29using System.Text;
[8598]30using HeuristicLab.Problems.DataAnalysis;
[8877]31using HeuristicLab.Random;
[8598]32
33namespace HeuristicLab.Problems.Instances.DataAnalysis {
[8877]34  public abstract class DataAnalysisInstanceProvider<TData, ImportType> : ProblemInstanceProvider<TData>
35    where TData : class, IDataAnalysisProblemData
36    where ImportType : DataAnalysisImportType {
[8598]37
[13414]38    public event ProgressChangedEventHandler ProgressChanged;
[8878]39
[8877]40    public TData ImportData(string path, ImportType type, DataAnalysisCSVFormat csvFormat) {
41      TableFileParser csvFileParser = new TableFileParser();
[14343]42      csvFileParser.Encoding = csvFormat.Encoding;
[13414]43      long fileSize = new FileInfo(path).Length;
44      csvFileParser.ProgressChanged += (sender, e) => {
45        OnProgressChanged(e / (double)fileSize);
46      };
[17414]47      var formatOptions = new TableFileFormatOptions {
48        NumberFormat = csvFormat.NumberFormatInfo, DateTimeFormat = csvFormat.DateTimeFormatInfo, ColumnSeparator = csvFormat.Separator
49      };
50      csvFileParser.Parse(path, formatOptions, csvFormat.VariableNamesAvailable);
[8877]51      return ImportData(path, type, csvFileParser);
52    }
[8598]53
[13414]54    protected virtual void OnProgressChanged(double d) {
55      var handler = ProgressChanged;
56      if (handler != null)
[13584]57        handler(this, new ProgressChangedEventArgs((int)(100 * d), null));
[13414]58    }
59
[8878]60    protected virtual TData ImportData(string path, ImportType type, TableFileParser csvFileParser) {
61      throw new NotSupportedException();
62    }
[8877]63
[8598]64    protected List<IList> Shuffle(List<IList> values) {
65      int count = values.First().Count;
[8877]66      int[] indices = Enumerable.Range(0, count).Shuffle(new FastRandom()).ToArray();
[9217]67      List<IList> shuffled = new List<IList>(values.Count);
[8598]68      for (int col = 0; col < values.Count; col++) {
69
70        if (values[col] is List<double>)
[9217]71          shuffled.Add(new List<double>());
[8598]72        else if (values[col] is List<DateTime>)
[9217]73          shuffled.Add(new List<DateTime>());
[8598]74        else if (values[col] is List<string>)
[9217]75          shuffled.Add(new List<string>());
[8598]76        else
77          throw new InvalidOperationException();
78
79        for (int i = 0; i < count; i++) {
[9217]80          shuffled[col].Add(values[col][indices[i]]);
[8598]81        }
82      }
[9217]83      return shuffled;
[8598]84    }
85
[8877]86    public override bool CanExportData {
87      get { return true; }
88    }
89    public override void ExportData(TData instance, string path) {
90      var strBuilder = new StringBuilder();
91      var colSep = CultureInfo.CurrentCulture.TextInfo.ListSeparator;
92      foreach (var variable in instance.Dataset.VariableNames) {
93        strBuilder.Append(variable.Replace(colSep, String.Empty) + colSep);
[8598]94      }
[8877]95      strBuilder.Remove(strBuilder.Length - colSep.Length, colSep.Length);
96      strBuilder.AppendLine();
97
98      var dataset = instance.Dataset;
99
100      for (int i = 0; i < dataset.Rows; i++) {
101        for (int j = 0; j < dataset.Columns; j++) {
102          if (j > 0) strBuilder.Append(colSep);
103          strBuilder.Append(dataset.GetValue(i, j));
104        }
105        strBuilder.AppendLine();
106      }
[13584]107      using (var fileStream = new FileStream(path, FileMode.Create)) {
[13901]108        Encoding encoding = Encoding.GetEncoding(Encoding.Default.CodePage,
109          new EncoderReplacementFallback("*"),
110          new DecoderReplacementFallback("*"));
111        using (var writer = new StreamWriter(fileStream, encoding)) {
[13584]112          writer.Write(strBuilder);
113        }
[8877]114      }
[8598]115    }
116  }
117}
Note: See TracBrowser for help on using the repository browser.