Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Clustering/CSV/ClusteringCSVInstanceProvider.cs @ 9608

Last change on this file since 9608 was 9608, checked in by sforsten, 11 years ago

#2070:

  • changed parse methods in TableFileParser to accept a bool which defines, if the first line contains variable names
  • added methods in TableFileParser to check if the first line contains variable names
  • adapted unit tests
  • adapted DataAnalysisImportTypeDialog so that a checkbox can be set to define if the first line contains variable names
  • added the flag NumberStyles.AllowTrailingSign for parsing doubles
File size: 4.8 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections;
24using System.Collections.Generic;
25using System.IO;
26using System.Linq;
27using HeuristicLab.Common;
28using HeuristicLab.Problems.DataAnalysis;
29
30namespace HeuristicLab.Problems.Instances.DataAnalysis {
31  public class ClusteringCSVInstanceProvider : ClusteringInstanceProvider {
32    public override string Name {
33      get { return "CSV File"; }
34    }
35    public override string Description {
36      get {
37        return "";
38      }
39    }
40    public override Uri WebLink {
41      get { return new Uri("http://dev.heuristiclab.com/trac/hl/core/wiki/UsersFAQ#DataAnalysisImportFileFormat"); }
42    }
43    public override string ReferencePublication {
44      get { return ""; }
45    }
46
47    public override IEnumerable<IDataDescriptor> GetDataDescriptors() {
48      return new List<IDataDescriptor>();
49    }
50
51    public override IClusteringProblemData LoadData(IDataDescriptor descriptor) {
52      throw new NotImplementedException();
53    }
54
55    public override bool CanImportData {
56      get { return true; }
57    }
58    public override IClusteringProblemData ImportData(string path) {
59      var csvFileParser = new TableFileParser();
60      csvFileParser.Parse(path, csvFileParser.AreColumnNamesInFirstLine(path));
61
62      Dataset dataset = new Dataset(csvFileParser.VariableNames, csvFileParser.Values);
63      string targetVar = dataset.DoubleVariables.Last();
64
65      // turn of input variables that are constant in the training partition
66      var allowedInputVars = new List<string>();
67      var trainingIndizes = Enumerable.Range(0, (csvFileParser.Rows * 2) / 3);
68      if (trainingIndizes.Count() >= 2) {
69        foreach (var variableName in dataset.DoubleVariables) {
70          if (dataset.GetDoubleValues(variableName, trainingIndizes).Range() > 0 &&
71            variableName != targetVar)
72            allowedInputVars.Add(variableName);
73        }
74      } else {
75        allowedInputVars.AddRange(dataset.DoubleVariables.Where(x => !x.Equals(targetVar)));
76      }
77
78      ClusteringProblemData clusteringData = new ClusteringProblemData(dataset, allowedInputVars);
79
80      int trainingPartEnd = trainingIndizes.Last();
81      clusteringData.TrainingPartition.Start = trainingIndizes.First();
82      clusteringData.TrainingPartition.End = trainingPartEnd;
83      clusteringData.TestPartition.Start = trainingPartEnd;
84      clusteringData.TestPartition.End = csvFileParser.Rows;
85
86      clusteringData.Name = Path.GetFileName(path);
87
88      return clusteringData;
89    }
90
91    protected override IClusteringProblemData ImportData(string path, DataAnalysisImportType type, TableFileParser csvFileParser) {
92      List<IList> values = csvFileParser.Values;
93      if (type.Shuffle) {
94        values = Shuffle(values);
95      }
96
97      Dataset dataset = new Dataset(csvFileParser.VariableNames, values);
98      string targetVar = dataset.DoubleVariables.Last();
99
100      // turn of input variables that are constant in the training partition
101      var allowedInputVars = new List<string>();
102      int trainingPartEnd = (csvFileParser.Rows * type.TrainingPercentage) / 100;
103      var trainingIndizes = Enumerable.Range(0, trainingPartEnd);
104      if (trainingIndizes.Count() >= 2) {
105        foreach (var variableName in dataset.DoubleVariables) {
106          if (dataset.GetDoubleValues(variableName, trainingIndizes).Range() > 0 &&
107            variableName != targetVar)
108            allowedInputVars.Add(variableName);
109        }
110      } else {
111        allowedInputVars.AddRange(dataset.DoubleVariables.Where(x => !x.Equals(targetVar)));
112      }
113
114      ClusteringProblemData clusteringData = new ClusteringProblemData(dataset, allowedInputVars);
115
116      clusteringData.TrainingPartition.Start = 0;
117      clusteringData.TrainingPartition.End = trainingPartEnd;
118      clusteringData.TestPartition.Start = trainingPartEnd;
119      clusteringData.TestPartition.End = csvFileParser.Rows;
120
121      clusteringData.Name = Path.GetFileName(path);
122
123      return clusteringData;
124    }
125  }
126}
Note: See TracBrowser for help on using the repository browser.