Changeset 8742 for branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Clustering/CSV

Timestamp:

10/05/12 11:58:17 (12 years ago)

Author:

mkommend

Message:

#1081: Merged trunk changes and fixed compilation errors due to the merge.

File:

• branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Clustering/CSV/ClusteringCSVInstanceProvider.cs (modified) (4 diffs)

branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Clustering/CSV/ClusteringCSVInstanceProvider.cs

@@ -21 +21 @@
 
 using System;
+using System.Collections;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
+using System.Linq;
 using System.Text;
+using HeuristicLab.Common;
 using HeuristicLab.Problems.DataAnalysis;
 
@@ -56 +60 @@
     public override IClusteringProblemData ImportData(string path) {
       var csvFileParser = new TableFileParser();
-
       csvFileParser.Parse(path);
 
-      var dataset = new Dataset(csvFileParser.VariableNames, csvFileParser.Values);
-      var claData = new ClusteringProblemData(dataset, dataset.DoubleVariables);
+      Dataset dataset = new Dataset(csvFileParser.VariableNames, csvFileParser.Values);
+      string targetVar = dataset.DoubleVariables.Last();
 
-      int trainingPartEnd = csvFileParser.Rows * 2 / 3;
-      claData.TrainingPartition.Start = 0;
-      claData.TrainingPartition.End = trainingPartEnd;
-      claData.TestPartition.Start = trainingPartEnd;
-      claData.TestPartition.End = csvFileParser.Rows;
-      int pos = path.LastIndexOf('\\');
-      if (pos < 0)
-        claData.Name = path;
-      else {
-        pos++;
-        claData.Name = path.Substring(pos, path.Length - pos);
+      // turn of input variables that are constant in the training partition
+      var allowedInputVars = new List<string>();
+      var trainingIndizes = Enumerable.Range(0, (csvFileParser.Rows * 2) / 3);
+      if (trainingIndizes.Count() >= 2) {
+        foreach (var variableName in dataset.DoubleVariables) {
+          if (dataset.GetDoubleValues(variableName, trainingIndizes).Range() > 0 &&
+            variableName != targetVar)
+            allowedInputVars.Add(variableName);
+        }
+      } else {
+        allowedInputVars.AddRange(dataset.DoubleVariables.Where(x => x.Equals(targetVar)));
       }
 
-      return claData;
+      ClusteringProblemData clusteringData = new ClusteringProblemData(dataset, allowedInputVars);
+
+      int trainingPartEnd = trainingIndizes.Last();
+      clusteringData.TrainingPartition.Start = trainingIndizes.First();
+      clusteringData.TrainingPartition.End = trainingPartEnd;
+      clusteringData.TestPartition.Start = trainingPartEnd;
+      clusteringData.TestPartition.End = csvFileParser.Rows;
+
+      clusteringData.Name = Path.GetFileName(path);
+
+      return clusteringData;
+    }
+
+    public override IClusteringProblemData ImportData(string path, DataAnalysisImportType type) {
+      TableFileParser csvFileParser = new TableFileParser();
+      csvFileParser.Parse(path);
+
+      List<IList> values = csvFileParser.Values;
+      if (type.Shuffle) {
+        values = Shuffle(values);
+      }
+
+      Dataset dataset = new Dataset(csvFileParser.VariableNames, values);
+      string targetVar = dataset.DoubleVariables.Last();
+
+      // turn of input variables that are constant in the training partition
+      var allowedInputVars = new List<string>();
+      int trainingPartEnd = (csvFileParser.Rows * type.Training) / 100;
+      var trainingIndizes = Enumerable.Range(0, trainingPartEnd);
+      foreach (var variableName in dataset.DoubleVariables) {
+        if (trainingIndizes.Count() >= 2 && dataset.GetDoubleValues(variableName, trainingIndizes).Range() > 0 &&
+          variableName != targetVar)
+          allowedInputVars.Add(variableName);
+      }
+
+      ClusteringProblemData clusteringData = new ClusteringProblemData(dataset, allowedInputVars);
+
+      clusteringData.TrainingPartition.Start = 0;
+      clusteringData.TrainingPartition.End = trainingPartEnd;
+      clusteringData.TestPartition.Start = trainingPartEnd;
+      clusteringData.TestPartition.End = csvFileParser.Rows;
+
+      clusteringData.Name = Path.GetFileName(path);
+
+      return clusteringData;
     }
 
@@ -83 +130 @@
     public override void ExportData(IClusteringProblemData instance, string path) {
       var strBuilder = new StringBuilder();
-
-      foreach (var variable in instance.InputVariables) {
-        strBuilder.Append(variable + ";");
+      var colSep = CultureInfo.CurrentCulture.TextInfo.ListSeparator;
+      foreach (var variable in instance.Dataset.VariableNames) {
+        strBuilder.Append(variable.Replace(colSep, String.Empty) + colSep);
       }
-      strBuilder.Remove(strBuilder.Length - 1, 1);
+      strBuilder.Remove(strBuilder.Length - colSep.Length, colSep.Length);
       strBuilder.AppendLine();
 
@@ -94 +141 @@
       for (int i = 0; i < dataset.Rows; i++) {
         for (int j = 0; j < dataset.Columns; j++) {
-          strBuilder.Append(dataset.GetValue(i, j) + ";");
+          if (j > 0) strBuilder.Append(colSep);
+          strBuilder.Append(dataset.GetValue(i, j));
         }
-        strBuilder.Remove(strBuilder.Length - 1, 1);
         strBuilder.AppendLine();
       }