Changeset 10624 for branches/DataPreprocessing

Timestamp:

03/19/14 14:02:37 (11 years ago)

Author:

mleitner

Message:

Filter undefined values on statistics

File:

• branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Implementations/StatisticsLogic.cs (modified) (9 diffs)

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Implementations/StatisticsLogic.cs

@@ -25 +25 @@
 using HeuristicLab.Common;
 
-namespace HeuristicLab.DataPreprocessing {
-
-  public class StatisticsLogic : IStatisticsLogic {
+namespace HeuristicLab.DataPreprocessing
+{
+
+  public class StatisticsLogic : IStatisticsLogic
+  {
 
     private readonly ITransactionalPreprocessingData preprocessingData;
     private readonly ISearchLogic searchLogic;
 
-    public StatisticsLogic(ITransactionalPreprocessingData thePreprocessingData, ISearchLogic theSearchLogic) {
+    public StatisticsLogic(ITransactionalPreprocessingData thePreprocessingData, ISearchLogic theSearchLogic)
+    {
       preprocessingData = thePreprocessingData;
       searchLogic = theSearchLogic;
     }
 
-    public int GetColumnCount() {
+    public int GetColumnCount()
+    {
       return preprocessingData.Columns;
     }
 
-    public int GetRowCount() {
+    public int GetRowCount()
+    {
       return preprocessingData.Rows;
     }
 
-    public int GetNumericColumnCount() {
+    public int GetNumericColumnCount()
+    {
       int count = 0;
 
-      for (int i = 0; i < preprocessingData.Columns; ++i) {
-        if (preprocessingData.IsType<double>(i)) {
+      for (int i = 0; i < preprocessingData.Columns; ++i)
+      {
+        if (preprocessingData.IsType<double>(i))
+        {
           ++count;
         }
@@ -56 +64 @@
     }
 
-    public int GetNominalColumnCount() {
+    public int GetNominalColumnCount()
+    {
       return preprocessingData.Columns - GetNumericColumnCount();
     }
 
-    public int GetMissingValueCount() {
+    public int GetMissingValueCount()
+    {
       int count = 0;
-      for (int i = 0; i < preprocessingData.Columns; ++i) {
+      for (int i = 0; i < preprocessingData.Columns; ++i)
+      {
         count += GetMissingValueCount(i);
       }
@@ -68 +79 @@
     }
 
-    public int GetMissingValueCount(int columnIndex) {
+    public int GetMissingValueCount(int columnIndex)
+    {
       return searchLogic.GetMissingValueIndices(columnIndex).Count();
     }
 
-    public T GetMin<T>(int columnIndex) where T : IComparable<T> {
+    public T GetMin<T>(int columnIndex) where T : IComparable<T>
+    {
       return preprocessingData.GetValues<T>(columnIndex).Min();
     }
 
-    public T GetMax<T>(int columnIndex) where T : IComparable<T> {
+    public T GetMax<T>(int columnIndex) where T : IComparable<T>
+    {
       return preprocessingData.GetValues<T>(columnIndex).Max();
     }
 
-    public double GetMedian(int columnIndex) {
+    public double GetMedian(int columnIndex)
+    {
       double median = double.NaN;
-      if (preprocessingData.IsType<double>(columnIndex)) {
-        median = preprocessingData.GetValues<double>(columnIndex).Median();
+      if (preprocessingData.IsType<double>(columnIndex))
+      {
+        median = GetValuesWithoutNaN<double>(columnIndex).Median();
       }
       return median;
     }
 
-    public double GetAverage(int columnIndex) {
+    public double GetAverage(int columnIndex)
+    {
       double avg = double.NaN;
-      if (preprocessingData.IsType<double>(columnIndex)) {
-        avg = preprocessingData.GetValues<double>(columnIndex).Average();
+      if (preprocessingData.IsType<double>(columnIndex))
+      {
+        avg = GetValuesWithoutNaN<double>(columnIndex).Where(x => !double.IsNaN(x)).Average();
       }
       return avg;
     }
 
-    public DateTime GetMedianDateTime(int columnIndex) {
+    public DateTime GetMedianDateTime(int columnIndex)
+    {
       DateTime median = new DateTime();
-      if (preprocessingData.IsType<DateTime>(columnIndex)) {
+      if (preprocessingData.IsType<DateTime>(columnIndex))
+      {
         median = GetSecondsAsDateTime(GetDateTimeAsSeconds(columnIndex).Median());
       }
@@ -104 +124 @@
     }
 
-    public DateTime GetAverageDateTime(int columnIndex) {
+    public DateTime GetAverageDateTime(int columnIndex)
+    {
       DateTime avg = new DateTime();
-      if (preprocessingData.IsType<DateTime>(columnIndex)) {
+      if (preprocessingData.IsType<DateTime>(columnIndex))
+      {
         avg = GetSecondsAsDateTime(GetDateTimeAsSeconds(columnIndex).Average());
       }
@@ -112 +134 @@
     }
 
-    public T GetMostCommonValue<T>(int columnIndex) {
+    public T GetMostCommonValue<T>(int columnIndex)
+    {
       var t = preprocessingData.GetValues<T>(columnIndex);
       var t2 = t.GroupBy(x => x);
@@ -125 +148 @@
 
 
-    public double GetStandardDeviation(int columnIndex) {
+    public double GetStandardDeviation(int columnIndex)
+    {
       double stdDev = double.NaN;
-      if (preprocessingData.IsType<double>(columnIndex)) {
-        stdDev = preprocessingData.GetValues<double>(columnIndex).StandardDeviation();
-      } else if (preprocessingData.IsType<DateTime>(columnIndex)) {
+      if (preprocessingData.IsType<double>(columnIndex))
+      {
+        stdDev = GetValuesWithoutNaN<double>(columnIndex).StandardDeviation();
+      }
+      else if (preprocessingData.IsType<DateTime>(columnIndex))
+      {
         stdDev = GetDateTimeAsSeconds(columnIndex).StandardDeviation();
       }
@@ -135 +162 @@
     }
 
-    public double GetVariance(int columnIndex) {
+    public double GetVariance(int columnIndex)
+    {
       double variance = double.NaN;
-      if (preprocessingData.IsType<double>(columnIndex)) {
+      if (preprocessingData.IsType<double>(columnIndex))
+      {
         variance = preprocessingData.GetValues<double>(columnIndex).Variance();
-      } else if (preprocessingData.IsType<DateTime>(columnIndex)) {
+      }
+      else if (preprocessingData.IsType<DateTime>(columnIndex))
+      {
         variance = GetDateTimeAsSeconds(columnIndex).Variance();
       }
@@ -145 +176 @@
     }
 
-    public int GetDifferentValuesCount<T>(int columnIndex) {
+    public int GetDifferentValuesCount<T>(int columnIndex)
+    {
       return preprocessingData.GetValues<T>(columnIndex).GroupBy(x => x).Count();
     }
 
-    public int GetRowMissingValueCount(int rowIndex) {
+    public int GetRowMissingValueCount(int rowIndex)
+    {
       int count = 0;
-      for (int i = 0; i < preprocessingData.Columns; ++i) {
-        if (searchLogic.IsMissingValue(i, rowIndex)) {
+      for (int i = 0; i < preprocessingData.Columns; ++i)
+      {
+        if (searchLogic.IsMissingValue(i, rowIndex))
+        {
           ++count;
         }
@@ -159 +194 @@
     }
 
-    public string GetVariableName(int columnIndex) {
+    public string GetVariableName(int columnIndex)
+    {
       return preprocessingData.GetVariableName(columnIndex);
     }
 
-    public bool IsType<T>(int columnIndex) {
+    public bool IsType<T>(int columnIndex)
+    {
       return preprocessingData.IsType<T>(columnIndex);
     }
 
-    public string GetColumnTypeAsString(int columnIndex) {
-      if (preprocessingData.IsType<double>(columnIndex)) {
+    public string GetColumnTypeAsString(int columnIndex)
+    {
+      if (preprocessingData.IsType<double>(columnIndex))
+      {
         return "double";
-      } else if (preprocessingData.IsType<string>(columnIndex)) {
+      }
+      else if (preprocessingData.IsType<string>(columnIndex))
+      {
         return "string";
-      } else if (preprocessingData.IsType<DateTime>(columnIndex)) {
+      }
+      else if (preprocessingData.IsType<DateTime>(columnIndex))
+      {
         return "DateTime";
       }
       return "Unknown Type";
     }
-    private IEnumerable<double> GetDateTimeAsSeconds(int columnIndex) {
-      return preprocessingData.GetValues<DateTime>(columnIndex).Select(x => (double)x.Ticks / TimeSpan.TicksPerSecond);
-    }
-
-    private DateTime GetSecondsAsDateTime(double seconds) {
+
+    private List<T> GetValuesWithoutNaN<T>(int columnIndex)
+    {
+      IEnumerable<int> missing = searchLogic.GetMissingValueIndices(columnIndex);
+      return (List<T>)preprocessingData.GetValues<T>(columnIndex).Select((v, i) => new { i, v }).Where(x => !missing.Contains(x.i));
+    }
+    private IEnumerable<double> GetDateTimeAsSeconds(int columnIndex)
+    {
+      return GetValuesWithoutNaN<DateTime>(columnIndex).Select(x => (double)x.Ticks / TimeSpan.TicksPerSecond);
+    }
+
+    private DateTime GetSecondsAsDateTime(double seconds)
+    {
       DateTime dateTime = new DateTime();
       return dateTime.AddSeconds(seconds);
     }
 
-    public event DataPreprocessingChangedEventHandler Changed {
+    public event DataPreprocessingChangedEventHandler Changed
+    {
       add { preprocessingData.Changed += value; }
       remove { preprocessingData.Changed -= value; }