Changeset 9363 for branches/OaaS/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/CSV

Timestamp:

04/16/13 13:13:41 (12 years ago)

Author:

spimming

Message:

#1888:

• Merged revisions from trunk

Location:

branches/OaaS

Files:

• . (modified) (2 props)
• HeuristicLab.Problems.Instances.DataAnalysis (modified) (1 prop)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/CSV/RegressionCSVInstanceProvider.cs (modified) (2 diffs)

branches/OaaS

@@ -21 +21 @@
 protoc.exe
 _ReSharper.HeuristicLab 3.3 Tests
+Google.ProtocolBuffers-2.4.1.473.dll
 packages

@@ -21 +21 @@
 protoc.exe
 _ReSharper.HeuristicLab 3.3 Tests
+Google.ProtocolBuffers-2.4.1.473.dll
 packages

branches/OaaS/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/CSV/RegressionCSVInstanceProvider.cs

@@ -21 +21 @@
 
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Text;
+using HeuristicLab.Common;
 using HeuristicLab.Problems.DataAnalysis;
 
@@ -59 +60 @@
 
       Dataset dataset = new Dataset(csvFileParser.VariableNames, csvFileParser.Values);
-      string targetVar = csvFileParser.VariableNames.Where(x => dataset.DoubleVariables.Contains(x)).Last();
+      string targetVar = dataset.DoubleVariables.Last();
 
-      IEnumerable<string> allowedInputVars = dataset.DoubleVariables.Where(x => !x.Equals(targetVar));
+      // turn of input variables that are constant in the training partition
+      var allowedInputVars = new List<string>();
+      var trainingIndizes = Enumerable.Range(0, (csvFileParser.Rows * 2) / 3);
+      if (trainingIndizes.Count() >= 2) {
+        foreach (var variableName in dataset.DoubleVariables) {
+          if (dataset.GetDoubleValues(variableName, trainingIndizes).Range() > 0 &&
+            variableName != targetVar)
+            allowedInputVars.Add(variableName);
+        }
+      } else {
+        allowedInputVars.AddRange(dataset.DoubleVariables.Where(x => !x.Equals(targetVar)));
+      }
 
-      IRegressionProblemData regData = new RegressionProblemData(dataset, allowedInputVars, targetVar);
+      IRegressionProblemData regressionData = new RegressionProblemData(dataset, allowedInputVars, targetVar);
 
-      int trainingPartEnd = csvFileParser.Rows * 2 / 3;
-      regData.TrainingPartition.Start = 0;
-      regData.TrainingPartition.End = trainingPartEnd;
-      regData.TestPartition.Start = trainingPartEnd;
-      regData.TestPartition.End = csvFileParser.Rows;
+      var trainingPartEnd = trainingIndizes.Last();
+      regressionData.TrainingPartition.Start = trainingIndizes.First();
+      regressionData.TrainingPartition.End = trainingPartEnd;
+      regressionData.TestPartition.Start = trainingPartEnd;
+      regressionData.TestPartition.End = csvFileParser.Rows;
 
-      int pos = path.LastIndexOf('\\');
-      if (pos < 0)
-        regData.Name = path;
-      else {
-        pos++;
-        regData.Name = path.Substring(pos, path.Length - pos);
-      }
-      return regData;
+      regressionData.Name = Path.GetFileName(path);
+
+      return regressionData;
     }
 
-    public override bool CanExportData {
-      get { return true; }
-    }
-    public override void ExportData(IRegressionProblemData instance, string path) {
-      StringBuilder strBuilder = new StringBuilder();
+    protected override IRegressionProblemData ImportData(string path, RegressionImportType type, TableFileParser csvFileParser) {
+      List<IList> values = csvFileParser.Values;
+      if (type.Shuffle) {
+        values = Shuffle(values);
+      }
+      Dataset dataset = new Dataset(csvFileParser.VariableNames, values);
 
-      foreach (var variable in instance.InputVariables) {
-        strBuilder.Append(variable + ";");
-      }
-      strBuilder.Remove(strBuilder.Length - 1, 1);
-      strBuilder.AppendLine();
-
-      Dataset dataset = instance.Dataset;
-
-      for (int i = 0; i < dataset.Rows; i++) {
-        for (int j = 0; j < dataset.Columns; j++) {
-          strBuilder.Append(dataset.GetValue(i, j) + ";");
+      // turn of input variables that are constant in the training partition
+      var allowedInputVars = new List<string>();
+      int trainingPartEnd = (csvFileParser.Rows * type.TrainingPercentage) / 100;
+      trainingPartEnd = trainingPartEnd > 0 ? trainingPartEnd : 1;
+      var trainingIndizes = Enumerable.Range(0, trainingPartEnd);
+      if (trainingIndizes.Count() >= 2) {
+        foreach (var variableName in dataset.DoubleVariables) {
+          if (dataset.GetDoubleValues(variableName, trainingIndizes).Range() > 0 &&
+            variableName != type.TargetVariable)
+            allowedInputVars.Add(variableName);
         }
-        strBuilder.Remove(strBuilder.Length - 1, 1);
-        strBuilder.AppendLine();
+      } else {
+        allowedInputVars.AddRange(dataset.DoubleVariables.Where(x => !x.Equals(type.TargetVariable)));
       }
 
-      using (StreamWriter writer = new StreamWriter(path)) {
-        writer.Write(strBuilder);
-      }
+      RegressionProblemData regressionData = new RegressionProblemData(dataset, allowedInputVars, type.TargetVariable);
+
+      regressionData.TrainingPartition.Start = 0;
+      regressionData.TrainingPartition.End = trainingPartEnd;
+      regressionData.TestPartition.Start = trainingPartEnd;
+      regressionData.TestPartition.End = csvFileParser.Rows;
+
+      regressionData.Name = Path.GetFileName(path);
+
+      return regressionData;
     }
   }

@@ -21 +21 @@
 protoc.exe
 _ReSharper.HeuristicLab 3.3 Tests
+Google.ProtocolBuffers-2.4.1.473.dll
 packages