Changeset 15283 for branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/PreprocessingData.cs

Timestamp:

07/24/17 15:17:35 (7 years ago)

Author:

pfleck

Message:

#2809: Removed StatisticsLogic.

File:

• branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/PreprocessingData.cs (modified) (3 diffs)

branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/PreprocessingData.cs

@@ -327 +327 @@
     }
 
+    public Type GetVariableType(int columnIndex) {
+      var listType = variableValues[columnIndex].GetType();
+      return listType.GenericTypeArguments.Single();
+    }
+
     public IList<string> InputVariables { get; private set; }
     public string TargetVariable { get; private set; } // optional
@@ -524 +529 @@
     #endregion
 
+    #region Statistics
+    public T GetMin<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
+      return values.Any() ? values.Min() : emptyValue;
+    }
+
+    public T GetMax<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
+      return values.Any() ? values.Max() : emptyValue;
+    }
+
+    public T GetMean<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+      if (typeof(T) == typeof(double)) {
+        var values = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
+        return values.Any() ? Convert<T>(values.Average()) : emptyValue;
+      }
+      if (typeof(T) == typeof(string)) {
+        return Convert<T>(string.Empty);
+      }
+      if (typeof(T) == typeof(DateTime)) {
+        var values = GetValuesWithoutMissingValues<DateTime>(columnIndex, considerSelection);
+        return values.Any() ? Convert<T>(AggregateAsDouble(values, Enumerable.Average)) : emptyValue;
+      }
+
+      throw new InvalidOperationException(typeof(T) + " not supported");
+    }
+
+    public T GetMedian<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IComparable<T> {
+      if (typeof(T) == typeof(double)) {// IEnumerable<double> is faster  
+        var doubleValues = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
+        return doubleValues.Any() ? Convert<T>(doubleValues.Median()) : emptyValue;
+      }
+      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
+      return values.Any() ? values.Quantile(0.5) : emptyValue;
+    }
+
+    public T GetMode<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IEquatable<T> {
+      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
+      return values.Any() ? values.GroupBy(x => x).OrderByDescending(g => g.Count()).Select(g => g.Key).First() : emptyValue;
+    }
+
+    public T GetStandardDeviation<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+      if (typeof(T) == typeof(double)) {
+        var values = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
+        return values.Any() ? Convert<T>(values.StandardDeviation()) : emptyValue;
+      }
+      // For DateTime, std.dev / variance would have to be TimeSpan
+      //if (typeof(T) == typeof(DateTime)) {
+      //  var values = GetValuesWithoutMissingValues<DateTime>(columnIndex, considerSelection);
+      //  return values.Any() ? Convert<T>(AggregateAsDouble(values, EnumerableStatisticExtensions.StandardDeviation)) : emptyValue;
+      //}
+      return default(T);
+    }
+
+    public T GetVariance<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+      if (typeof(T) == typeof(double)) {
+        var values = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
+        return values.Any() ? Convert<T>(values.Variance()) : emptyValue;
+      }
+      // DateTime variance often overflows long, thus the corresponding DateTime is invalid
+      //if (typeof(T) == typeof(DateTime)) {
+      //  var values = GetValuesWithoutMissingValues<DateTime>(columnIndex, considerSelection);
+      //  return values.Any() ? Convert<T>(AggregateAsDouble(values, EnumerableStatisticExtensions.Variance)) : emptyValue;
+      //}
+      return default(T);
+    }
+
+    public T GetQuantile<T>(double alpha, int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IComparable<T> {
+      if (typeof(T) == typeof(double)) {// IEnumerable<double> is faster  
+        var doubleValues = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
+        return doubleValues.Any() ? Convert<T>(doubleValues.Quantile(alpha)) : emptyValue;
+      }
+      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
+      return values.Any() ? values.Quantile(alpha) : emptyValue;
+    }
+
+    public int GetDistinctValues<T>(int columnIndex, bool considerSelection = false) {
+      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
+      return values.GroupBy(x => x).Count();
+    }
+
+    private IEnumerable<T> GetValuesWithoutMissingValues<T>(int columnIndex, bool considerSelection) {
+      return GetValues<T>(columnIndex, considerSelection).Where(x => !IsMissingValue(x));
+    }
+
+    private static DateTime AggregateAsDouble(IEnumerable<DateTime> values, Func<IEnumerable<double>, double> func) {
+      return new DateTime((long)(func(values.Select(x => (double)x.Ticks / TimeSpan.TicksPerSecond)) * TimeSpan.TicksPerSecond));
+    }
+    private static T Convert<T>(object obj) { return (T)obj; }
+
+    public int GetMissingValueCount() {
+      int count = 0;
+      for (int i = 0; i < Columns; ++i) {
+        count += GetMissingValueCount(i);
+      }
+      return count;
+    }
+    public int GetMissingValueCount(int columnIndex) {
+      int sum = 0;
+      for (int i = 0; i < Rows; i++) {
+        if (IsCellEmpty(columnIndex, i))
+          sum++;
+      }
+      return sum;
+    }
+    public int GetRowMissingValueCount(int rowIndex) {
+      int sum = 0;
+      for (int i = 0; i < Columns; i++) {
+        if (IsCellEmpty(i, rowIndex))
+          sum++;
+      }
+      return sum;
+    }
+    #endregion
+
     #region Helpers
     private static IList<IList> CopyVariableValues(IList<IList> original) {
@@ -534 +654 @@
     #endregion
   }
+
+  // Adapted from HeuristicLab.Common.EnumerableStatisticExtensions
+  internal static class EnumerableExtensions {
+    public static T Quantile<T>(this IEnumerable<T> values, double alpha) where T : IComparable<T> {
+      T[] valuesArr = values.ToArray();
+      int n = valuesArr.Length;
+      if (n == 0) throw new InvalidOperationException("Enumeration contains no elements.");
+
+      var pos = n * alpha;
+
+      return Select((int)Math.Ceiling(pos) - 1, valuesArr);
+
+    }
+
+    private static T Select<T>(int k, T[] arr) where T : IComparable<T> {
+      int i, ir, j, l, mid, n = arr.Length;
+      T a;
+      l = 0;
+      ir = n - 1;
+      for (;;) {
+        if (ir <= l + 1) {
+          // Active partition contains 1 or 2 elements.
+          if (ir == l + 1 && arr[ir].CompareTo(arr[l]) < 0) {
+            // Case of 2 elements.
+            Swap(arr, l, ir);
+          }
+          return arr[k];
+        } else {
+          mid = (l + ir) >> 1; // Choose median of left, center, and right elements
+          Swap(arr, mid, l + 1); // as partitioning element a. Also
+
+          if (arr[l].CompareTo(arr[ir]) > 0) {  // rearrange so that arr[l] arr[ir] <= arr[l+1],
+            Swap(arr, l, ir); // . arr[ir] >= arr[l+1]
+          }
+
+          if (arr[l + 1].CompareTo(arr[ir]) > 0) {
+            Swap(arr, l + 1, ir);
+          }
+          if (arr[l].CompareTo(arr[l + 1]) > 0) {
+            Swap(arr, l, l + 1);
+          }
+          i = l + 1; // Initialize pointers for partitioning.
+          j = ir;
+          a = arr[l + 1]; // Partitioning element.
+          for (;;) { // Beginning of innermost loop.
+            do i++; while (arr[i].CompareTo(a) < 0); // Scan up to find element > a.
+            do j--; while (arr[j].CompareTo(a) > 0); // Scan down to find element < a.
+            if (j < i) break; // Pointers crossed. Partitioning complete.
+            Swap(arr, i, j);
+          } // End of innermost loop.
+          arr[l + 1] = arr[j]; // Insert partitioning element.
+          arr[j] = a;
+          if (j >= k) ir = j - 1; // Keep active the partition that contains the
+          if (j <= k) l = i; // kth element.
+        }
+      }
+    }
+
+    private static void Swap<T>(T[] arr, int i, int j) {
+      T temp = arr[i];
+      arr[i] = arr[j];
+      arr[j] = temp;
+    }
+  }
 }