source: trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/ResourceClassificationInstanceProvider.cs @ 7965

Last change on this file since 7965 was 7965, checked in by sforsten, 9 years ago

#1784: CSV files that contain columns with non-numeric data can be imported again.

File size: 4.0 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Globalization;
25using System.IO;
26using System.Linq;
27using System.Reflection;
28using System.Text.RegularExpressions;
29using HeuristicLab.Problems.DataAnalysis;
30using ICSharpCode.SharpZipLib.Zip;
31
32namespace HeuristicLab.Problems.Instances.DataAnalysis {
33  public abstract class ResourceClassificationInstanceProvider : ClassificationInstanceProvider {
34
35    protected abstract string FileName { get; }
36
37    public override IEnumerable<IDataDescriptor> GetDataDescriptors() {
38      var solutionsArchiveName = GetResourceName(FileName + @"\.zip");
39      if (!String.IsNullOrEmpty(solutionsArchiveName)) {
40        using (var solutionsZipFile = new ZipInputStream(GetType().Assembly.GetManifestResourceStream(solutionsArchiveName))) {
41          IList<string> entries = new List<string>();
42          ZipEntry curEntry;
43          while ((curEntry = solutionsZipFile.GetNextEntry()) != null) {
44            entries.Add(curEntry.Name);
45          }
46          foreach (var entry in entries.OrderBy(x => x)) {
47            yield return new ResourceClassificationDataDescriptor(Path.GetFileNameWithoutExtension(entry), Description, entry);
48          }
49        }
50      }
51    }
52
53    public override IClassificationProblemData LoadData(IDataDescriptor id) {
54      var descriptor = (ResourceClassificationDataDescriptor)id;
55
56      var instanceArchiveName = GetResourceName(FileName + @"\.zip");
57      using (var instancesZipFile = new ZipFile(GetType().Assembly.GetManifestResourceStream(instanceArchiveName))) {
58        var entry = instancesZipFile.GetEntry(descriptor.ResourceName);
59        NumberFormatInfo numberFormat;
60        DateTimeFormatInfo dateFormat;
61        char separator;
62        using (Stream stream = instancesZipFile.GetInputStream(entry)) {
63          TableFileParser.DetermineFileFormat(stream, out numberFormat, out dateFormat, out separator);
64        }
65
66        TableFileParser csvFileParser = new TableFileParser();
67        using (Stream stream = instancesZipFile.GetInputStream(entry)) {
68          csvFileParser.Parse(stream, numberFormat, dateFormat, separator);
69        }
70
71        Dataset dataset = new Dataset(csvFileParser.VariableNames, csvFileParser.Values);
72        string targetVar = csvFileParser.VariableNames.Where(x => dataset.DoubleVariables.Contains(x)).Last();
73        IEnumerable<string> allowedInputVars = dataset.DoubleVariables.Where(x => !x.Equals(targetVar));
74
75        ClassificationProblemData claData = new ClassificationProblemData(dataset, allowedInputVars, targetVar);
76
77        int trainingPartEnd = csvFileParser.Rows * 2 / 3;
78        claData.TrainingPartition.Start = 0;
79        claData.TrainingPartition.End = trainingPartEnd;
80        claData.TestPartition.Start = trainingPartEnd;
81        claData.TestPartition.End = csvFileParser.Rows;
82
83        claData.Name = descriptor.Name;
84        claData.Description = descriptor.Description;
85        return claData;
86      }
87    }
88
89    protected virtual string GetResourceName(string fileName) {
90      return Assembly.GetExecutingAssembly().GetManifestResourceNames()
91              .Where(x => Regex.Match(x, @".*\.Data\." + fileName).Success).SingleOrDefault();
92    }
93  }
94}
Note: See TracBrowser for help on using the repository browser.