Changeset 8585 for branches/HeuristicLab.Mono/HeuristicLab.Problems.Instances.DataAnalysis/3.3

Timestamp:

09/06/12 09:52:52 (13 years ago)

Author:

ascheibe

Message:

#1861 merged changes from trunk into branch

Location:

branches/HeuristicLab.Mono

Files:

• . (modified) (1 prop)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/CSV/ClassifiactionCSVInstanceProvider.cs (modified) (3 diffs)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/Clustering/CSV/ClusteringCSVInstanceProvider.cs (modified) (4 diffs)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/Plugin.cs.frame (modified) (1 diff)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/CSV/RegressionCSVInstanceProvider.cs (modified) (3 diffs)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/TableFileParser.cs (modified) (1 diff)

branches/HeuristicLab.Mono/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/CSV/ClassifiactionCSVInstanceProvider.cs

@@ -22 +22 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Text;
+using HeuristicLab.Common;
 using HeuristicLab.Problems.DataAnalysis;
 
@@ -61 +63 @@
 
       Dataset dataset = new Dataset(csvFileParser.VariableNames, csvFileParser.Values);
-      string targetVar = csvFileParser.VariableNames.Where(x => dataset.DoubleVariables.Contains(x)).Last();
-      IEnumerable<string> allowedInputVars = dataset.DoubleVariables.Where(x => !x.Equals(targetVar));
+      string targetVar = dataset.DoubleVariables.Last();
 
-      ClassificationProblemData claData = new ClassificationProblemData(dataset, allowedInputVars, targetVar);
-
-      int trainingPartEnd = csvFileParser.Rows * 2 / 3;
-      claData.TrainingPartition.Start = 0;
-      claData.TrainingPartition.End = trainingPartEnd;
-      claData.TestPartition.Start = trainingPartEnd;
-      claData.TestPartition.End = csvFileParser.Rows;
-      int pos = path.LastIndexOf('\\');
-      if (pos < 0)
-        claData.Name = path;
-      else {
-        pos++;
-        claData.Name = path.Substring(pos, path.Length - pos);
+      // turn of input variables that are constant in the training partition
+      var allowedInputVars = new List<string>();
+      var trainingIndizes = Enumerable.Range(0, (csvFileParser.Rows * 2) / 3);
+      foreach (var variableName in dataset.DoubleVariables) {
+        if (dataset.GetDoubleValues(variableName, trainingIndizes).Range() > 0 &&
+          variableName != targetVar)
+          allowedInputVars.Add(variableName);
       }
 
-      return claData;
+      ClassificationProblemData classificationData = new ClassificationProblemData(dataset, allowedInputVars, targetVar);
+
+      int trainingPartEnd = trainingIndizes.Last();
+      classificationData.TrainingPartition.Start = trainingIndizes.First();
+      classificationData.TrainingPartition.End = trainingPartEnd;
+      classificationData.TestPartition.Start = trainingPartEnd;
+      classificationData.TestPartition.End = csvFileParser.Rows;
+
+      classificationData.Name = Path.GetFileName(path);
+
+      return classificationData;
     }
 
@@ -86 +91 @@
     }
     public override void ExportData(IClassificationProblemData instance, string path) {
-      StringBuilder strBuilder = new StringBuilder();
+      var strBuilder = new StringBuilder();
 
       foreach (var variable in instance.InputVariables) {
-        strBuilder.Append(variable + ";");
+        strBuilder.Append(variable + CultureInfo.CurrentCulture.TextInfo.ListSeparator);
       }
-      strBuilder.Remove(strBuilder.Length - 1, 1);
+      strBuilder.Remove(strBuilder.Length - CultureInfo.CurrentCulture.TextInfo.ListSeparator.Length, CultureInfo.CurrentCulture.TextInfo.ListSeparator.Length);
       strBuilder.AppendLine();
 
-      Dataset dataset = instance.Dataset;
+      var dataset = instance.Dataset;
 
       for (int i = 0; i < dataset.Rows; i++) {
         for (int j = 0; j < dataset.Columns; j++) {
-          strBuilder.Append(dataset.GetValue(i, j) + ";");
+          if (j > 0) strBuilder.Append(CultureInfo.CurrentCulture.TextInfo.ListSeparator);
+          strBuilder.Append(dataset.GetValue(i, j));
         }
-        strBuilder.Remove(strBuilder.Length - 1, 1);
         strBuilder.AppendLine();
       }
 
-      using (StreamWriter writer = new StreamWriter(path)) {
+      using (var writer = new StreamWriter(path)) {
         writer.Write(strBuilder);
       }

branches/HeuristicLab.Mono/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Clustering/CSV/ClusteringCSVInstanceProvider.cs

@@ -22 +22 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
+using System.Linq;
 using System.Text;
+using HeuristicLab.Common;
 using HeuristicLab.Problems.DataAnalysis;
 
@@ -60 +63 @@
 
       var dataset = new Dataset(csvFileParser.VariableNames, csvFileParser.Values);
-      var claData = new ClusteringProblemData(dataset, dataset.DoubleVariables);
 
-      int trainingPartEnd = csvFileParser.Rows * 2 / 3;
-      claData.TrainingPartition.Start = 0;
-      claData.TrainingPartition.End = trainingPartEnd;
-      claData.TestPartition.Start = trainingPartEnd;
-      claData.TestPartition.End = csvFileParser.Rows;
-      int pos = path.LastIndexOf('\\');
-      if (pos < 0)
-        claData.Name = path;
-      else {
-        pos++;
-        claData.Name = path.Substring(pos, path.Length - pos);
+      // turn of input variables that are constant in the training partition
+      var allowedInputVars = new List<string>();
+      var trainingIndizes = Enumerable.Range(0, (csvFileParser.Rows * 2) / 3);
+      foreach (var variableName in dataset.DoubleVariables) {
+        if (dataset.GetDoubleValues(variableName, trainingIndizes).Range() > 0)
+          allowedInputVars.Add(variableName);
       }
 
-      return claData;
+      var clusteringData = new ClusteringProblemData(dataset, allowedInputVars);
+
+      int trainingPartEnd = trainingIndizes.Last();
+      clusteringData.TrainingPartition.Start = trainingIndizes.First();
+      clusteringData.TrainingPartition.End = trainingPartEnd;
+      clusteringData.TestPartition.Start = trainingPartEnd;
+      clusteringData.TestPartition.End = csvFileParser.Rows;
+
+      clusteringData.Name = Path.GetFileName(path);
+
+      return clusteringData;
     }
 
@@ -85 +92 @@
 
       foreach (var variable in instance.InputVariables) {
-        strBuilder.Append(variable + ";");
+        strBuilder.Append(variable + CultureInfo.CurrentCulture.TextInfo.ListSeparator);
       }
-      strBuilder.Remove(strBuilder.Length - 1, 1);
+      strBuilder.Remove(strBuilder.Length - CultureInfo.CurrentCulture.TextInfo.ListSeparator.Length, CultureInfo.CurrentCulture.TextInfo.ListSeparator.Length);
       strBuilder.AppendLine();
 
@@ -94 +101 @@
       for (int i = 0; i < dataset.Rows; i++) {
         for (int j = 0; j < dataset.Columns; j++) {
-          strBuilder.Append(dataset.GetValue(i, j) + ";");
+          if (j > 0) strBuilder.Append(CultureInfo.CurrentCulture.TextInfo.ListSeparator);
+          strBuilder.Append(dataset.GetValue(i, j));
         }
-        strBuilder.Remove(strBuilder.Length - 1, 1);
         strBuilder.AppendLine();
       }

branches/HeuristicLab.Mono/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Plugin.cs.frame

@@ -25 +25 @@
   [Plugin("HeuristicLab.Problems.Instances.DataAnalysis", "3.3.7.$WCREV$")]
   [PluginFile("HeuristicLab.Problems.Instances.DataAnalysis-3.3.dll", PluginFileType.Assembly)]
+  [PluginDependency("HeuristicLab.Common", "3.3")]
   [PluginDependency("HeuristicLab.Core", "3.3")]
   [PluginDependency("HeuristicLab.Data", "3.3")]

branches/HeuristicLab.Mono/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/CSV/RegressionCSVInstanceProvider.cs

@@ -22 +22 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Text;
+using HeuristicLab.Common;
 using HeuristicLab.Problems.DataAnalysis;
 
@@ -59 +61 @@
 
       Dataset dataset = new Dataset(csvFileParser.VariableNames, csvFileParser.Values);
-      string targetVar = csvFileParser.VariableNames.Where(x => dataset.DoubleVariables.Contains(x)).Last();
+      string targetVar = dataset.DoubleVariables.Last();
 
-      IEnumerable<string> allowedInputVars = dataset.DoubleVariables.Where(x => !x.Equals(targetVar));
+      // turn of input variables that are constant in the training partition
+      var allowedInputVars = new List<string>();
+      var trainingIndizes = Enumerable.Range(0, (csvFileParser.Rows * 2) / 3);
+      foreach (var variableName in dataset.DoubleVariables) {
+        if (dataset.GetDoubleValues(variableName, trainingIndizes).Range() > 0 &&
+          variableName != targetVar)
+          allowedInputVars.Add(variableName);
+      }
 
-      IRegressionProblemData regData = new RegressionProblemData(dataset, allowedInputVars, targetVar);
+      IRegressionProblemData regressionData = new RegressionProblemData(dataset, allowedInputVars, targetVar);
 
-      int trainingPartEnd = csvFileParser.Rows * 2 / 3;
-      regData.TrainingPartition.Start = 0;
-      regData.TrainingPartition.End = trainingPartEnd;
-      regData.TestPartition.Start = trainingPartEnd;
-      regData.TestPartition.End = csvFileParser.Rows;
+      var trainingPartEnd = trainingIndizes.Last();
+      regressionData.TrainingPartition.Start = trainingIndizes.First();
+      regressionData.TrainingPartition.End = trainingPartEnd;
+      regressionData.TestPartition.Start = trainingPartEnd;
+      regressionData.TestPartition.End = csvFileParser.Rows;
 
-      int pos = path.LastIndexOf('\\');
-      if (pos < 0)
-        regData.Name = path;
-      else {
-        pos++;
-        regData.Name = path.Substring(pos, path.Length - pos);
-      }
-      return regData;
+      regressionData.Name = Path.GetFileName(path);
+
+      return regressionData;
     }
 
@@ -85 +89 @@
     }
     public override void ExportData(IRegressionProblemData instance, string path) {
-      StringBuilder strBuilder = new StringBuilder();
+      var strBuilder = new StringBuilder();
 
       foreach (var variable in instance.InputVariables) {
-        strBuilder.Append(variable + ";");
+        strBuilder.Append(variable + CultureInfo.CurrentCulture.TextInfo.ListSeparator);
       }
-      strBuilder.Remove(strBuilder.Length - 1, 1);
+      strBuilder.Remove(strBuilder.Length - CultureInfo.CurrentCulture.TextInfo.ListSeparator.Length, CultureInfo.CurrentCulture.TextInfo.ListSeparator.Length);
       strBuilder.AppendLine();
 
-      Dataset dataset = instance.Dataset;
+      var dataset = instance.Dataset;
 
       for (int i = 0; i < dataset.Rows; i++) {
         for (int j = 0; j < dataset.Columns; j++) {
-          strBuilder.Append(dataset.GetValue(i, j) + ";");
+          if (j > 0) strBuilder.Append(CultureInfo.CurrentCulture.TextInfo.ListSeparator);
+          strBuilder.Append(dataset.GetValue(i, j));
         }
-        strBuilder.Remove(strBuilder.Length - 1, 1);
         strBuilder.AppendLine();
       }
 
-      using (StreamWriter writer = new StreamWriter(path)) {
+      using (var writer = new StreamWriter(path)) {
         writer.Write(strBuilder);
       }

branches/HeuristicLab.Mono/HeuristicLab.Problems.Instances.DataAnalysis/3.3/TableFileParser.cs

@@ -32 +32 @@
 namespace HeuristicLab.Problems.Instances.DataAnalysis {
   public class TableFileParser {
-    private const int BUFFER_SIZE = 1024;
+    private const int BUFFER_SIZE = 65536;
     private static readonly char[] POSSIBLE_SEPARATORS = new char[] { ',', ';', '\t' };
     private Tokenizer tokenizer;