Changeset 9363 for branches/OaaS/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification

Timestamp:

04/16/13 13:13:41 (11 years ago)

Author:

spimming

Message:

#1888:

• Merged revisions from trunk

Location:

branches/OaaS

Files:

• . (modified) (2 props)
• HeuristicLab.Problems.Instances.DataAnalysis (modified) (1 prop)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/ArtificialClassificationDataDescriptor.cs (modified) (2 diffs)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/CSV/ClassifiactionCSVInstanceProvider.cs (modified) (2 diffs)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/ClassificationDataDescriptor.cs (copied) (copied from trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/ClassificationDataDescriptor.cs)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/ClassificationImportType.cs (copied) (copied from trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/ClassificationImportType.cs)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/ClassificationInstanceProvider.cs (modified) (1 diff)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/Data/ClassificationRealWorld.zip (deleted)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/Data/UCI.zip (copied) (copied from trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/Data/UCI.zip)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/RealWorld (deleted)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/ResourceClassificationDataDescriptor.cs (modified) (1 diff)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/ResourceClassificationInstanceProvider.cs (modified) (3 diffs)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/UCI (copied) (copied from trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/UCI)

branches/OaaS

@@ -21 +21 @@
 protoc.exe
 _ReSharper.HeuristicLab 3.3 Tests
+Google.ProtocolBuffers-2.4.1.473.dll
 packages

@@ -21 +21 @@
 protoc.exe
 _ReSharper.HeuristicLab 3.3 Tests
+Google.ProtocolBuffers-2.4.1.473.dll
 packages

branches/OaaS/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/ArtificialClassificationDataDescriptor.cs

@@ -29 +29 @@
 
     protected abstract string TargetVariable { get; }
-    protected abstract string[] InputVariables { get; }
+    protected abstract string[] VariableNames { get; }
     protected abstract string[] AllowedInputVariables { get; }
     protected abstract int TrainingPartitionStart { get; }
@@ -37 +37 @@
 
     public IClassificationProblemData GenerateClassificationData() {
-      Dataset dataset = new Dataset(InputVariables, this.GenerateValues());
+      Dataset dataset = new Dataset(VariableNames, this.GenerateValues());
 
       ClassificationProblemData claData = new ClassificationProblemData(dataset, AllowedInputVariables, TargetVariable);

branches/OaaS/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/CSV/ClassifiactionCSVInstanceProvider.cs

@@ -21 +21 @@
 
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Text;
+using HeuristicLab.Common;
 using HeuristicLab.Problems.DataAnalysis;
 
@@ -61 +62 @@
 
       Dataset dataset = new Dataset(csvFileParser.VariableNames, csvFileParser.Values);
-      string targetVar = csvFileParser.VariableNames.Where(x => dataset.DoubleVariables.Contains(x)).Last();
-      IEnumerable<string> allowedInputVars = dataset.DoubleVariables.Where(x => !x.Equals(targetVar));
+      string targetVar = dataset.DoubleVariables.Last();
 
-      ClassificationProblemData claData = new ClassificationProblemData(dataset, allowedInputVars, targetVar);
-
-      int trainingPartEnd = csvFileParser.Rows * 2 / 3;
-      claData.TrainingPartition.Start = 0;
-      claData.TrainingPartition.End = trainingPartEnd;
-      claData.TestPartition.Start = trainingPartEnd;
-      claData.TestPartition.End = csvFileParser.Rows;
-      int pos = path.LastIndexOf('\\');
-      if (pos < 0)
-        claData.Name = path;
-      else {
-        pos++;
-        claData.Name = path.Substring(pos, path.Length - pos);
+      // turn of input variables that are constant in the training partition
+      var allowedInputVars = new List<string>();
+      var trainingIndizes = Enumerable.Range(0, (csvFileParser.Rows * 2) / 3);
+      if (trainingIndizes.Count() >= 2) {
+        foreach (var variableName in dataset.DoubleVariables) {
+          if (dataset.GetDoubleValues(variableName, trainingIndizes).Range() > 0 &&
+            variableName != targetVar)
+            allowedInputVars.Add(variableName);
+        }
+      } else {
+        allowedInputVars.AddRange(dataset.DoubleVariables.Where(x => !x.Equals(targetVar)));
       }
 
-      return claData;
+      ClassificationProblemData classificationData = new ClassificationProblemData(dataset, allowedInputVars, targetVar);
+
+      int trainingPartEnd = trainingIndizes.Last();
+      classificationData.TrainingPartition.Start = trainingIndizes.First();
+      classificationData.TrainingPartition.End = trainingPartEnd;
+      classificationData.TestPartition.Start = trainingPartEnd;
+      classificationData.TestPartition.End = csvFileParser.Rows;
+
+      classificationData.Name = Path.GetFileName(path);
+
+      return classificationData;
     }
 
-    public override bool CanExportData {
-      get { return true; }
-    }
-    public override void ExportData(IClassificationProblemData instance, string path) {
-      StringBuilder strBuilder = new StringBuilder();
-
-      foreach (var variable in instance.InputVariables) {
-        strBuilder.Append(variable + ";");
-      }
-      strBuilder.Remove(strBuilder.Length - 1, 1);
-      strBuilder.AppendLine();
-
-      Dataset dataset = instance.Dataset;
-
-      for (int i = 0; i < dataset.Rows; i++) {
-        for (int j = 0; j < dataset.Columns; j++) {
-          strBuilder.Append(dataset.GetValue(i, j) + ";");
+    protected override IClassificationProblemData ImportData(string path, ClassificationImportType type, TableFileParser csvFileParser) {
+      int trainingPartEnd = (csvFileParser.Rows * type.TrainingPercentage) / 100;
+      List<IList> values = csvFileParser.Values;
+      if (type.Shuffle) {
+        values = Shuffle(values);
+        if (type.UniformlyDistributeClasses) {
+          values = Shuffle(values, csvFileParser.VariableNames.ToList().FindIndex(x => x.Equals(type.TargetVariable)),
+                           type.TrainingPercentage, out trainingPartEnd);
         }
-        strBuilder.Remove(strBuilder.Length - 1, 1);
-        strBuilder.AppendLine();
       }
 
-      using (StreamWriter writer = new StreamWriter(path)) {
-        writer.Write(strBuilder);
+      Dataset dataset = new Dataset(csvFileParser.VariableNames, values);
+
+      // turn of input variables that are constant in the training partition
+      var allowedInputVars = new List<string>();
+      var trainingIndizes = Enumerable.Range(0, trainingPartEnd);
+      if (trainingIndizes.Count() >= 2) {
+        foreach (var variableName in dataset.DoubleVariables) {
+          if (dataset.GetDoubleValues(variableName, trainingIndizes).Range() > 0 &&
+            variableName != type.TargetVariable)
+            allowedInputVars.Add(variableName);
+        }
+      } else {
+        allowedInputVars.AddRange(dataset.DoubleVariables.Where(x => !x.Equals(type.TargetVariable)));
       }
+
+      ClassificationProblemData classificationData = new ClassificationProblemData(dataset, allowedInputVars, type.TargetVariable);
+
+      classificationData.TrainingPartition.Start = 0;
+      classificationData.TrainingPartition.End = trainingPartEnd;
+      classificationData.TestPartition.Start = trainingPartEnd;
+      classificationData.TestPartition.End = csvFileParser.Rows;
+
+      classificationData.Name = Path.GetFileName(path);
+
+      return classificationData;
+    }
+
+    protected List<IList> Shuffle(List<IList> values, int target, int trainingPercentage, out int trainingPartEnd) {
+      IList targetValues = values[target];
+      var group = targetValues.Cast<double>().GroupBy(x => x).Select(g => new { Key = g.Key, Count = g.Count() }).ToList();
+      Dictionary<double, double> taken = new Dictionary<double, double>();
+      foreach (var classCount in group) {
+        taken[classCount.Key] = (classCount.Count * trainingPercentage) / 100.0;
+      }
+
+      List<IList> training = GetListOfIListCopy(values);
+      List<IList> test = GetListOfIListCopy(values);
+
+      for (int i = 0; i < targetValues.Count; i++) {
+        if (taken[(double)targetValues[i]] > 0) {
+          AddRow(training, values, i);
+          taken[(double)targetValues[i]]--;
+        } else {
+          AddRow(test, values, i);
+        }
+      }
+
+      trainingPartEnd = training.First().Count;
+
+      for (int i = 0; i < training.Count; i++) {
+        for (int j = 0; j < test[i].Count; j++) {
+          training[i].Add(test[i][j]);
+        }
+      }
+
+      return training;
+    }
+
+    private void AddRow(List<IList> destination, List<IList> source, int index) {
+      for (int i = 0; i < source.Count; i++) {
+        destination[i].Add(source[i][index]);
+      }
+    }
+
+    private List<IList> GetListOfIListCopy(List<IList> values) {
+      List<IList> newList = new List<IList>(values.Count);
+      foreach (IList t in values) {
+        if (t is List<double>)
+          newList.Add(new List<double>());
+        else if (t is List<DateTime>)
+          newList.Add(new List<DateTime>());
+        else if (t is List<string>)
+          newList.Add(new List<string>());
+        else
+          throw new InvalidOperationException();
+      }
+      return newList;
     }
   }

branches/OaaS/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/ClassificationInstanceProvider.cs

@@ -23 +23 @@
 
 namespace HeuristicLab.Problems.Instances.DataAnalysis {
-  public abstract class ClassificationInstanceProvider : ProblemInstanceProvider<IClassificationProblemData> {
+  public abstract class ClassificationInstanceProvider : DataAnalysisInstanceProvider<IClassificationProblemData, ClassificationImportType> {
   }
 }

branches/OaaS/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/ResourceClassificationDataDescriptor.cs

@@ -20 +20 @@
 #endregion
 
+using System.Collections.Generic;
+using System.Linq;
 
 namespace HeuristicLab.Problems.Instances.DataAnalysis {
-  internal class ResourceClassificationDataDescriptor : IDataDescriptor {
-    public string Name { get; internal set; }
-    public string Description { get; internal set; }
+  public abstract class ResourceClassificationDataDescriptor : ClassificationDataDescriptor {
+    internal string ResourceName { get; set; }
 
-    internal string ResourceName { get; set; }
-    internal ResourceClassificationDataDescriptor(string name, string description, string resourceName) {
-      Name = name;
-      Description = description;
-      ResourceName = resourceName;
+    public bool CheckVariableNames(IEnumerable<string> VariableNames) {
+      return this.VariableNames.All(x => VariableNames.Contains(x));
     }
   }

branches/OaaS/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/ResourceClassificationInstanceProvider.cs

@@ -21 +21 @@
 
 using System;
-using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
@@ -34 +33 @@
 
     protected abstract string FileName { get; }
-
-    public override IEnumerable<IDataDescriptor> GetDataDescriptors() {
-      var solutionsArchiveName = GetResourceName(FileName + @"\.zip");
-      if (!String.IsNullOrEmpty(solutionsArchiveName)) {
-        using (var solutionsZipFile = new ZipInputStream(GetType().Assembly.GetManifestResourceStream(solutionsArchiveName))) {
-          IList<string> entries = new List<string>();
-          ZipEntry curEntry;
-          while ((curEntry = solutionsZipFile.GetNextEntry()) != null) {
-            entries.Add(curEntry.Name);
-          }
-          foreach (var entry in entries.OrderBy(x => x)) {
-            yield return new ResourceClassificationDataDescriptor(Path.GetFileNameWithoutExtension(entry), Description, entry);
-          }
-        }
-      }
-    }
 
     public override IClassificationProblemData LoadData(IDataDescriptor id) {
@@ -70 +53 @@
 
         Dataset dataset = new Dataset(csvFileParser.VariableNames, csvFileParser.Values);
-        string targetVar = csvFileParser.VariableNames.Where(x => dataset.DoubleVariables.Contains(x)).Last();
-        IEnumerable<string> allowedInputVars = dataset.DoubleVariables.Where(x => !x.Equals(targetVar));
+        if (!descriptor.CheckVariableNames(csvFileParser.VariableNames)) {
+          throw new ArgumentException("Parsed file contains variables which are not in the descriptor.");
+        }
 
-        ClassificationProblemData claData = new ClassificationProblemData(dataset, allowedInputVars, targetVar);
-
-        int trainingPartEnd = csvFileParser.Rows * 2 / 3;
-        claData.TrainingPartition.Start = 0;
-        claData.TrainingPartition.End = trainingPartEnd;
-        claData.TestPartition.Start = trainingPartEnd;
-        claData.TestPartition.End = csvFileParser.Rows;
-
-        claData.Name = descriptor.Name;
-        claData.Description = descriptor.Description;
-        return claData;
+        return descriptor.GenerateClassificationData(dataset);
       }
     }

@@ -21 +21 @@
 protoc.exe
 _ReSharper.HeuristicLab 3.3 Tests
+Google.ProtocolBuffers-2.4.1.473.dll
 packages