Changeset 14843 for trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/DatasetExtensions.cs

Timestamp:

04/11/17 15:55:44 (7 years ago)

Author:

gkronber

Message:

#2697: applied r14390, r14391, r14393, r14394, r14396 again (resolving conflicts)

File:

• trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/DatasetExtensions.cs (modified) (1 diff)

trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/DatasetExtensions.cs

@@ -20 +20 @@
 #endregion
 
+using System;
 using System.Collections.Generic;
+using System.Linq;
 
 namespace HeuristicLab.Problems.DataAnalysis {
   public static class DatasetExtensions {
-    public static IEnumerable<T> TakeEvery<T>(this IEnumerable<T> xs, int nth) {
-      int i = 0;
-      foreach (var x in xs) {
-        if (i % nth == 0) yield return x;
-        i++;
+    public static double[,] ToArray(this IDataset dataset, IEnumerable<string> variables, IEnumerable<int> rows) {
+      return ToArray(dataset,
+        variables,
+        transformations: variables.Select(_ => (ITransformation<double>)null), // no transform
+        rows: rows);
+    }
+    public static double[,] ToArray(this IDataset dataset, IEnumerable<string> variables,
+      IEnumerable<ITransformation<double>> transformations, IEnumerable<int> rows) {
+      string[] variablesArr = variables.ToArray();
+      int[] rowsArr = rows.ToArray();
+      ITransformation<double>[] transformArr = transformations.ToArray();
+      if (transformArr.Length != variablesArr.Length)
+        throw new ArgumentException("Number of variables and number of transformations must match.");
+
+      double[,] matrix = new double[rowsArr.Length, variablesArr.Length];
+
+      for (int i = 0; i < variablesArr.Length; i++) {
+        var origValues = dataset.GetDoubleValues(variablesArr[i], rowsArr);
+        var values = transformArr[i] != null ? transformArr[i].Apply(origValues) : origValues;
+        int row = 0;
+        foreach (var value in values) {
+          matrix[row, i] = value;
+          row++;
+        }
       }
+
+      return matrix;
+    }
+
+    /// <summary>
+    /// Prepares a binary data matrix from a number of factors and specified factor values
+    /// </summary>
+    /// <param name="dataset">A dataset that contains the variable values</param>
+    /// <param name="factorVariables">An enumerable of categorical variables (factors). For each variable an enumerable of values must be specified.</param>
+    /// <param name="rows">An enumerable of row indices for the dataset</param>
+    /// <returns></returns>
+    /// <remarks>Factor variables (categorical variables) are split up into multiple binary variables one for each specified value.</remarks>
+    public static double[,] ToArray(
+      this IDataset dataset,
+      IEnumerable<KeyValuePair<string, IEnumerable<string>>> factorVariables,
+      IEnumerable<int> rows) {
+      // check input variables. Only string variables are allowed.
+      var invalidInputs =
+        factorVariables.Select(kvp => kvp.Key).Where(name => !dataset.VariableHasType<string>(name));
+      if (invalidInputs.Any())
+        throw new NotSupportedException("Unsupported inputs: " + string.Join(", ", invalidInputs));
+
+      int numBinaryColumns = factorVariables.Sum(kvp => kvp.Value.Count());
+
+      List<int> rowsList = rows.ToList();
+      double[,] matrix = new double[rowsList.Count, numBinaryColumns];
+
+      int col = 0;
+      foreach (var kvp in factorVariables) {
+        var varName = kvp.Key;
+        var cats = kvp.Value;
+        if (!cats.Any()) continue;
+        foreach (var cat in cats) {
+          var values = dataset.GetStringValues(varName, rows);
+          int row = 0;
+          foreach (var value in values) {
+            matrix[row, col] = value == cat ? 1 : 0;
+            row++;
+          }
+          col++;
+        }
+      }
+      return matrix;
+    }
+
+    public static IEnumerable<KeyValuePair<string, IEnumerable<string>>> GetFactorVariableValues(
+      this IDataset ds, IEnumerable<string> factorVariables, IEnumerable<int> rows) {
+      return from factor in factorVariables
+             let distinctValues = ds.GetStringValues(factor, rows).Distinct().ToArray()
+             // 1 distinct value => skip (constant)
+             // 2 distinct values => only take one of the two values
+             // >=3 distinct values => create a binary value for each value
+             let reducedValues = distinctValues.Length <= 2
+               ? distinctValues.Take(distinctValues.Length - 1)
+               : distinctValues
+             select new KeyValuePair<string, IEnumerable<string>>(factor, reducedValues);
     }
   }