Changeset 10661 for branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3

Timestamp:

03/26/14 13:34:18 (11 years ago)

Author:

mleitner

Message:

Cache Missing value dedection

Location:

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3

Files:

• Implementations/SearchLogic.cs (modified) (3 diffs)
• Implementations/StatisticsLogic.cs (modified) (2 diffs)
• Interfaces/ISearchLogic.cs (modified) (1 diff)

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Implementations/SearchLogic.cs

@@ -21 +21 @@
 
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
@@ -28 +29 @@
     private readonly ITransactionalPreprocessingData preprocessingData;
 
+    private Dictionary<int, IEnumerable<int>> MissingValueIndicies { get; set; }
+    private Dictionary<int, IEnumerable> ValuesWithoutNaN { get; set; }
+
     public SearchLogic(ITransactionalPreprocessingData thePreprocessingData) {
       preprocessingData = thePreprocessingData;
+
+      MissingValueIndicies = new Dictionary<int, IEnumerable<int>>();
+      ValuesWithoutNaN = new Dictionary<int, IEnumerable>();
+
+      preprocessingData.Changed += preprocessingData_Changed;
+    }
+
+    void preprocessingData_Changed(object sender, DataPreprocessingChangedEventArgs e)
+    {
+      MissingValueIndicies.Remove(e.Column);
+      ValuesWithoutNaN.Remove(e.Column);
     }
 
@@ -53 +68 @@
 
     public IEnumerable<int> GetMissingValueIndices(int columnIndex) {
-      if (preprocessingData.IsType<double>(columnIndex)) {
-        return preprocessingData.GetValues<double>(columnIndex).Select((s, i) => new { i, s }).Where(t => double.IsNaN(t.s)).Select(t => t.i);
-      } else if (preprocessingData.IsType<string>(columnIndex)) {
-        return preprocessingData.GetValues<string>(columnIndex).Select((s, i) => new { i, s }).Where(t => string.IsNullOrEmpty(t.s)).Select(t => t.i);
-      } else if (preprocessingData.IsType<DateTime>(columnIndex)) {
-        return preprocessingData.GetValues<DateTime>(columnIndex).Select((s, i) => new { i, s }).Where(t => t.s.Equals(DateTime.MinValue)).Select(t => t.i);
-      } else {
-        throw new ArgumentException("column " + columnIndex + " contains a non supported type.");
+      if (!MissingValueIndicies.ContainsKey(columnIndex)){        
+          if (preprocessingData.IsType<double>(columnIndex)) {
+            MissingValueIndicies[columnIndex] = GetMissingValueIndices<double>(columnIndex);
+          } else if (preprocessingData.IsType<string>(columnIndex)) {
+            MissingValueIndicies[columnIndex] = GetMissingValueIndices<string>(columnIndex);
+          } else if (preprocessingData.IsType<DateTime>(columnIndex)) {
+            MissingValueIndicies[columnIndex] = GetMissingValueIndices<DateTime>(columnIndex);
+          } else {
+            throw new ArgumentException("column " + columnIndex + " contains a non supported type.");
+          }
+      } 
+
+      return MissingValueIndicies[columnIndex];
+   }
+    private IEnumerable<int> GetMissingValueIndices<T>(int columnIndex) {
+      List<int> missingIndices = new List<int>();
+     
+      for(int row = 0; row < preprocessingData.Rows; ++row) {
+        if (IsMissingValue(columnIndex, row)) {
+          missingIndices.Add(row);
+        }
       }
+
+      return missingIndices;
     }
 
+    public IEnumerable<T> GetValuesWithoutNaN<T>(int columnIndex)
+    {
+      if (!ValuesWithoutNaN.ContainsKey(columnIndex))
+      {
+        List<T> values = new List<T>();
+
+        for (int row = 0; row < preprocessingData.Rows; ++row)
+        {
+          if (!IsMissingValue(columnIndex, row))
+          {
+            values.Add(preprocessingData.GetCell<T>(columnIndex, row));
+          }
+        }
+
+        ValuesWithoutNaN[columnIndex] = values;
+      }
+
+      return (IEnumerable<T>)ValuesWithoutNaN[columnIndex];
+    }
   }
 }

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Implementations/StatisticsLogic.cs

@@ -24 +24 @@
 using System.Linq;
 using HeuristicLab.Common;
+using System.Collections;
 
 namespace HeuristicLab.DataPreprocessing {
@@ -178 +179 @@
     }
 
-    private List<T> GetValuesWithoutNaN<T>(int columnIndex) {
-      IEnumerable<int> missing = searchLogic.GetMissingValueIndices(columnIndex);
-      return preprocessingData.GetValues<T>(columnIndex)
-        .Select((v, i) => new { i, v })
-        .Where(x => !missing.Contains(x.i))
-        .Select(x => x.v).ToList<T>();
-    }
     private IEnumerable<double> GetDateTimeAsSeconds(int columnIndex) {
       return GetValuesWithoutNaN<DateTime>(columnIndex).Select(x => (double)x.Ticks / TimeSpan.TicksPerSecond);
+    }
+
+    private IEnumerable<T> GetValuesWithoutNaN<T>(int columnIndex) {
+      return searchLogic.GetValuesWithoutNaN<T>(columnIndex);
     }
 

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Interfaces/ISearchLogic.cs

@@ -38 +38 @@
 
     bool IsMissingValue(int columnIndex, int rowIndex);
+
+    IEnumerable<T> GetValuesWithoutNaN<T>(int columnIndex);
   }
 }