Changeset 10809 for branches/DataPreprocessing/HeuristicLab.DataPreprocessing

Timestamp:

05/07/14 12:47:54 (10 years ago)

Author:

sbreuer

Message:

• selected average and co. implemented
• SelectionChanged NullPointer fixed

Location:

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3

Files:

• Implementations/ChartLogic.cs (modified) (2 diffs)
• Implementations/DataGridContent.cs (modified) (1 diff)
• Implementations/FilteredPreprocessingData.cs (modified) (1 diff)
• Implementations/ManipulationLogic.cs (modified) (8 diffs)
• Implementations/PreprocessingData.cs (modified) (1 diff)
• Implementations/SearchLogic.cs (modified) (4 diffs)
• Implementations/StatisticsLogic.cs (modified) (5 diffs)
• Interfaces/IDataGridContent.cs (modified) (1 diff)
• Interfaces/IManipulationLogic.cs (modified) (1 diff)
• Interfaces/IPreprocessingData.cs (modified) (1 diff)
• Interfaces/ISearchLogic.cs (modified) (1 diff)
• Interfaces/IStatisticsLogic.cs (modified) (1 diff)
• PreprocessingTransformator.cs (modified) (1 diff)

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Implementations/ChartLogic.cs

@@ -37 +37 @@
 
     public DataRow CreateDataRow(string variableName, DataRowVisualProperties.DataRowChartType chartType) {
-      IList<double> values = preprocessingData.GetValues<double>(variableName);
+      IList<double> values = preprocessingData.GetValues<double>(variableName, false);
       DataRow row = new DataRow(variableName, "", values);
       row.VisualProperties.ChartType = chartType;
@@ -44 +44 @@
 
     public DataRow CreateDataRowRange(string variableName,int start, int end, DataRowVisualProperties.DataRowChartType chartType) {
-      IList<double> values = preprocessingData.GetValues<double>(variableName);
+      IList<double> values = preprocessingData.GetValues<double>(variableName, false);
       IList<double> valuesRange = new List<double>();
       for (int i = 0; i < values.Count; i++) {

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Implementations/DataGridContent.cs

@@ -45 +45 @@
     }
 
-    public IManipulationLogic PreprocessingDataManipulation {
+    public IManipulationLogic ManipulationLogic {
       get { return manipulationLogic; }
     }

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Implementations/FilteredPreprocessingData.cs

@@ -41 +41 @@
     }
 
-    public IList<T> GetValues<T>(string variableName) {
-      return ActiveData.GetValues<T>(variableName);
+    public IList<T> GetValues<T>(string variableName, bool considerSelection) {
+      return ActiveData.GetValues<T>(variableName, considerSelection);
     }
 
-    public IList<T> GetValues<T>(int columnIndex) {
-      return ActiveData.GetValues<T>(columnIndex);
+    public IList<T> GetValues<T>(int columnIndex, bool considerSelection) {
+      return ActiveData.GetValues<T>(columnIndex, considerSelection);
     }
 

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Implementations/ManipulationLogic.cs

@@ -45 +45 @@
     }
 
-    public void ReplaceIndicesByAverageValue(IDictionary<int, IList<int>> cells) {
-      preprocessingData.InTransaction(() => {
-        foreach (var column in cells) {
-          if (preprocessingData.IsType<double>(column.Key)) {
-            double average = statisticsLogic.GetAverage(column.Key);
+    public void ReplaceIndicesByAverageValue(IDictionary<int, IList<int>> cells, bool considerSelection) {
+      preprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          if (preprocessingData.IsType<double>(column.Key)) {
+            double average = statisticsLogic.GetAverage(column.Key, considerSelection);
             ReplaceIndicesByValue<double>(column.Key, column.Value, average);
           } else if (preprocessingData.IsType<DateTime>(column.Key)) {
-            DateTime average = statisticsLogic.GetAverageDateTime(column.Key);
+            DateTime average = statisticsLogic.GetAverageDateTime(column.Key, considerSelection);
             ReplaceIndicesByValue<DateTime>(column.Key, column.Value, average);
           }
@@ -59 +59 @@
     }
 
-    public void ReplaceIndicesByMedianValue(IDictionary<int, IList<int>> cells) {
-      preprocessingData.InTransaction(() => {
-        foreach (var column in cells) {
-          if (preprocessingData.IsType<double>(column.Key)) {
-            double median = statisticsLogic.GetMedian(column.Key);
+    public void ReplaceIndicesByMedianValue(IDictionary<int, IList<int>> cells, bool considerSelection) {
+      preprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          if (preprocessingData.IsType<double>(column.Key)) {
+            double median = statisticsLogic.GetMedian(column.Key, considerSelection);
             ReplaceIndicesByValue<double>(column.Key, column.Value, median);
           } else if (preprocessingData.IsType<DateTime>(column.Key)) {
-            DateTime median = statisticsLogic.GetMedianDateTime(column.Key);
+            DateTime median = statisticsLogic.GetMedianDateTime(column.Key, considerSelection);
             ReplaceIndicesByValue<DateTime>(column.Key, column.Value, median);
           }
@@ -73 +73 @@
     }
 
-    public void ReplaceIndicesByRandomValue(IDictionary<int, IList<int>> cells) {
+    public void ReplaceIndicesByRandomValue(IDictionary<int, IList<int>> cells, bool considerSelection) {
       preprocessingData.InTransaction(() => {
         Random r = new Random();
@@ -79 +79 @@
         foreach (var column in cells) {
           if (preprocessingData.IsType<double>(column.Key)) {
-            double max = statisticsLogic.GetMax<double>(column.Key);
-            double min = statisticsLogic.GetMin<double>(column.Key);
+            double max = statisticsLogic.GetMax<double>(column.Key, considerSelection);
+            double min = statisticsLogic.GetMin<double>(column.Key, considerSelection);
             double randMultiplier = (max - min);
             foreach (int index in column.Value) {
@@ -87 +87 @@
             }
           } else if (preprocessingData.IsType<DateTime>(column.Key)) {
-            DateTime min = statisticsLogic.GetMin<DateTime>(column.Key);
-            DateTime max = statisticsLogic.GetMax<DateTime>(column.Key);
+            DateTime min = statisticsLogic.GetMin<DateTime>(column.Key, considerSelection);
+            DateTime max = statisticsLogic.GetMax<DateTime>(column.Key, considerSelection);
             double randMultiplier = (max - min).TotalSeconds;
             foreach (int index in column.Value) {
@@ -104 +104 @@
           int countValues = 0;
           if (preprocessingData.IsType<double>(column.Key)) {
-            countValues = preprocessingData.GetValues<double>(column.Key).Count();
-          } else if (preprocessingData.IsType<DateTime>(column.Key)) {
-            countValues = preprocessingData.GetValues<DateTime>(column.Key).Count();
+            countValues = preprocessingData.GetValues<double>(column.Key, false).Count();
+          } else if (preprocessingData.IsType<DateTime>(column.Key)) {
+            countValues = preprocessingData.GetValues<DateTime>(column.Key, false).Count();
           }
 
@@ -165 +165 @@
     }
 
-    public void ReplaceIndicesByMostCommonValue(IDictionary<int, IList<int>> cells) {
-      preprocessingData.InTransaction(() => {
-        foreach (var column in cells) {
-          if (preprocessingData.IsType<double>(column.Key)) {
-            ReplaceIndicesByValue<double>(column.Key, column.Value, statisticsLogic.GetMostCommonValue<double>(column.Key));
+    public void ReplaceIndicesByMostCommonValue(IDictionary<int, IList<int>> cells, bool considerSelection) {
+      preprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          if (preprocessingData.IsType<double>(column.Key)) {
+            ReplaceIndicesByValue<double>(column.Key, column.Value, statisticsLogic.GetMostCommonValue<double>(column.Key, considerSelection));
           } else if (preprocessingData.IsType<string>(column.Key)) {
-            ReplaceIndicesByValue<string>(column.Key, column.Value, statisticsLogic.GetMostCommonValue<string>(column.Key));
-          } else if (preprocessingData.IsType<DateTime>(column.Key)) {
-            ReplaceIndicesByValue<DateTime>(column.Key, column.Value, statisticsLogic.GetMostCommonValue<DateTime>(column.Key));
+            ReplaceIndicesByValue<string>(column.Key, column.Value, statisticsLogic.GetMostCommonValue<string>(column.Key, considerSelection));
+          } else if (preprocessingData.IsType<DateTime>(column.Key)) {
+            ReplaceIndicesByValue<DateTime>(column.Key, column.Value, statisticsLogic.GetMostCommonValue<DateTime>(column.Key, considerSelection));
           } else {
             throw new ArgumentException("column with index: " + column.Key + " contains a non supported type.");
@@ -256 +256 @@
     private void reOrderToIndices<T>(int columnIndex, IList<Tuple<int, int>> indices) {
 
-      List<T> originalData = new List<T>(preprocessingData.GetValues<T>(columnIndex));
+      List<T> originalData = new List<T>(preprocessingData.GetValues<T>(columnIndex, false));
 
       // process all columns equally

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Implementations/PreprocessingData.cs

@@ -122 +122 @@
 
     [Obsolete("use the index based variant, is faster")]
-    public IList<T> GetValues<T>(string variableName) {
-      return GetValues<T>(GetColumnIndex(variableName));
-    }
-
-    public IList<T> GetValues<T>(int columnIndex) {
-      return (IList<T>)variableValues[columnIndex];
+    public IList<T> GetValues<T>(string variableName, bool considerSelection) {
+      return GetValues<T>(GetColumnIndex(variableName), considerSelection);
+    }
+
+    public IList<T> GetValues<T>(int columnIndex, bool considerSelection) {
+      if (considerSelection) {
+        var list = new List<T>();
+        foreach (var rowIdx in currentSelection[columnIndex]) {
+          list.Add((T)variableValues[columnIndex][rowIdx]);
+        }
+        return list;
+      } else {
+        return (IList<T>)variableValues[columnIndex];
+      }
     }
 

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Implementations/SearchLogic.cs

@@ -30 +30 @@
 
     private Dictionary<int, IList<int>> MissingValueIndicies { get; set; }
-    private Dictionary<int, IEnumerable> ValuesWithoutNaN { get; set; }
+    private Dictionary<int, IList> ValuesWithoutNaN { get; set; }
 
     public SearchLogic(ITransactionalPreprocessingData thePreprocessingData) {
@@ -36 +36 @@
 
       MissingValueIndicies = new Dictionary<int, IList<int>>();
-      ValuesWithoutNaN = new Dictionary<int, IEnumerable>();
+      ValuesWithoutNaN = new Dictionary<int, IList>();
 
       preprocessingData.Changed += preprocessingData_Changed;
@@ -58 +58 @@
         case DataPreprocessingChangedEventType.Transformation:
           MissingValueIndicies = new Dictionary<int, IList<int>>();
-          ValuesWithoutNaN = new Dictionary<int, IEnumerable>();
+          ValuesWithoutNaN = new Dictionary<int, IList>();
           break;
       } 
@@ -110 +110 @@
     }
 
-    public IEnumerable<T> GetValuesWithoutNaN<T>(int columnIndex)
+    public IEnumerable<T> GetValuesWithoutNaN<T>(int columnIndex, bool considerSelection)
     {
-      if (!ValuesWithoutNaN.ContainsKey(columnIndex))
-      {
+      if (considerSelection) {     
+        var selectedRows =  preprocessingData.GetSelection()[columnIndex];
+        
         List<T> values = new List<T>();
-
-        for (int row = 0; row < preprocessingData.Rows; ++row)
-        {
-          if (!IsMissingValue(columnIndex, row))
-          {
-            values.Add(preprocessingData.GetCell<T>(columnIndex, row));
+        foreach (var rowIdx in selectedRows) {
+          if (!IsMissingValue(columnIndex, rowIdx)) {
+            values.Add(preprocessingData.GetCell<T>(columnIndex, rowIdx));
           }
         }
+        return values;
+      } else {
+        if (!ValuesWithoutNaN.ContainsKey(columnIndex)) {
+          List<T> values = new List<T>();
 
-        ValuesWithoutNaN[columnIndex] = values;
+          for (int row = 0; row < preprocessingData.Rows; ++row) {
+            if (!IsMissingValue(columnIndex, row)) {
+              values.Add(preprocessingData.GetCell<T>(columnIndex, row));
+            }
+          }
+
+          ValuesWithoutNaN[columnIndex] = values;
+        }
+        return (IEnumerable<T>)ValuesWithoutNaN[columnIndex];
       }
-
-      return (IEnumerable<T>)ValuesWithoutNaN[columnIndex];
     }
   }

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Implementations/StatisticsLogic.cs

@@ -73 +73 @@
     }
 
-    public T GetMin<T>(int columnIndex) where T : IComparable<T> {
-      return preprocessingData.GetValues<T>(columnIndex).Min();
+    public T GetMin<T>(int columnIndex, bool considerSelection) where T : IComparable<T> {
+      return preprocessingData.GetValues<T>(columnIndex, considerSelection).Min();
     }
 
-    public T GetMax<T>(int columnIndex) where T : IComparable<T> {
-      return preprocessingData.GetValues<T>(columnIndex).Max();
+    public T GetMax<T>(int columnIndex, bool considerSelection) where T : IComparable<T> {
+      return preprocessingData.GetValues<T>(columnIndex, considerSelection).Max();
     }
 
-    public double GetMedian(int columnIndex) {
+    public double GetMedian(int columnIndex, bool considerSelection) {
       double median = double.NaN;
       if (preprocessingData.IsType<double>(columnIndex)) {
-        median = GetValuesWithoutNaN<double>(columnIndex).Median();
+        median = GetValuesWithoutNaN<double>(columnIndex, considerSelection).Median();
       }
       return median;
     }
 
-    public double GetAverage(int columnIndex) {
+    public double GetAverage(int columnIndex, bool considerSelection) {
       double avg = double.NaN;
       if (preprocessingData.IsType<double>(columnIndex)) {
-        avg = GetValuesWithoutNaN<double>(columnIndex).Average();
+        avg = GetValuesWithoutNaN<double>(columnIndex, considerSelection).Average();
       }
       return avg;
     }
 
-    public DateTime GetMedianDateTime(int columnIndex) {
+    public DateTime GetMedianDateTime(int columnIndex, bool considerSelection) {
       DateTime median = new DateTime();
       if (preprocessingData.IsType<DateTime>(columnIndex)) {
-        median = GetSecondsAsDateTime(GetDateTimeAsSeconds(columnIndex).Median());
+        median = GetSecondsAsDateTime(GetDateTimeAsSeconds(columnIndex, considerSelection).Median());
       }
       return median;
     }
 
-    public DateTime GetAverageDateTime(int columnIndex) {
+    public DateTime GetAverageDateTime(int columnIndex, bool considerSelection) {
       DateTime avg = new DateTime();
       if (preprocessingData.IsType<DateTime>(columnIndex)) {
-        avg = GetSecondsAsDateTime(GetDateTimeAsSeconds(columnIndex).Average());
+        avg = GetSecondsAsDateTime(GetDateTimeAsSeconds(columnIndex, considerSelection).Average());
       }
       return avg;
     }
 
-    public T GetMostCommonValue<T>(int columnIndex) {
-      var t = preprocessingData.GetValues<T>(columnIndex);
+    public T GetMostCommonValue<T>(int columnIndex, bool considerSelection) {
+      var t = preprocessingData.GetValues<T>(columnIndex, considerSelection);
       var t2 = t.GroupBy(x => x);
       var t3 = t2.Select(g => g.Key);
 
-      return preprocessingData.GetValues<T>(columnIndex)
+      return preprocessingData.GetValues<T>(columnIndex, considerSelection)
                               .GroupBy(x => x)
                               .OrderByDescending(g => g.Count())
@@ -129 +129 @@
       double stdDev = double.NaN;
       if (preprocessingData.IsType<double>(columnIndex)) {
-        stdDev = GetValuesWithoutNaN<double>(columnIndex).StandardDeviation();
+        stdDev = GetValuesWithoutNaN<double>(columnIndex, false).StandardDeviation();
       } else if (preprocessingData.IsType<DateTime>(columnIndex)) {
-        stdDev = GetDateTimeAsSeconds(columnIndex).StandardDeviation();
+        stdDev = GetDateTimeAsSeconds(columnIndex, false).StandardDeviation();
       }
       return stdDev;
@@ -139 +139 @@
       double variance = double.NaN;
       if (preprocessingData.IsType<double>(columnIndex)) {
-        variance = preprocessingData.GetValues<double>(columnIndex).Variance();
+        variance = preprocessingData.GetValues<double>(columnIndex, false).Variance();
       } else if (preprocessingData.IsType<DateTime>(columnIndex)) {
-        variance = GetDateTimeAsSeconds(columnIndex).Variance();
+        variance = GetDateTimeAsSeconds(columnIndex, false).Variance();
       }
       return variance;
@@ -147 +147 @@
 
     public int GetDifferentValuesCount<T>(int columnIndex) {
-      return preprocessingData.GetValues<T>(columnIndex).GroupBy(x => x).Count();
+      return preprocessingData.GetValues<T>(columnIndex, false).GroupBy(x => x).Count();
     }
 
@@ -179 +179 @@
     }
 
-    private IEnumerable<double> GetDateTimeAsSeconds(int columnIndex) {
-      return GetValuesWithoutNaN<DateTime>(columnIndex).Select(x => (double)x.Ticks / TimeSpan.TicksPerSecond);
+    private IEnumerable<double> GetDateTimeAsSeconds(int columnIndex, bool considerSelection) {
+      return GetValuesWithoutNaN<DateTime>(columnIndex, considerSelection).Select(x => (double)x.Ticks / TimeSpan.TicksPerSecond);
     }
 
-    private IEnumerable<T> GetValuesWithoutNaN<T>(int columnIndex) {
-      return searchLogic.GetValuesWithoutNaN<T>(columnIndex);
+    private IEnumerable<T> GetValuesWithoutNaN<T>(int columnIndex, bool considerSelection) {
+      return searchLogic.GetValuesWithoutNaN<T>(columnIndex, considerSelection);
     }
 

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Interfaces/IDataGridContent.cs

@@ -25 +25 @@
   public interface IDataGridContent : IStringConvertibleMatrix {
     IDataGridLogic DataGridLogic { get; }
-    IManipulationLogic PreprocessingDataManipulation { get; }
+    IManipulationLogic ManipulationLogic { get; }
     IFilterLogic FilterLogic { get; }
 

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Interfaces/IManipulationLogic.cs

@@ -28 +28 @@
     void ReOrderToIndices(IList<Tuple<int, int>> indices);
     void ShuffleToIndices(IList<System.Tuple<int, int>> indices);
-    void ReplaceIndicesByAverageValue(IDictionary<int, IList<int>> cells);
+    void ReplaceIndicesByAverageValue(IDictionary<int, IList<int>> cells, bool considerSelection);
     void ReplaceIndicesByLinearInterpolationOfNeighbours(IDictionary<int, IList<int>> cells);
-    void ReplaceIndicesByMedianValue(IDictionary<int, IList<int>> cells);
-    void ReplaceIndicesByMostCommonValue(IDictionary<int, IList<int>> cells);
-    void ReplaceIndicesByRandomValue(IDictionary<int, IList<int>> cells);
+    void ReplaceIndicesByMedianValue(IDictionary<int, IList<int>> cells, bool considerSelection);
+    void ReplaceIndicesByMostCommonValue(IDictionary<int, IList<int>> cells, bool considerSelection);
+    void ReplaceIndicesByRandomValue(IDictionary<int, IList<int>> cells, bool considerSelection);
     void ReplaceIndicesByValue(IDictionary<int, IList<int>> cells, string value);
     void ReplaceIndicesByValue<T>(int columnIndex, IEnumerable<int> rowIndices, T value);

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Interfaces/IPreprocessingData.cs

@@ -37 +37 @@
 
     [Obsolete("use the index based variant, is faster")]
-    IList<T> GetValues<T>(string variableName);
-    IList<T> GetValues<T>(int columnIndex);
+    IList<T> GetValues<T>(string variableName, bool considerSelection);
+    IList<T> GetValues<T>(int columnIndex, bool considerSelection);
 
     void SetValues<T>(int columnIndex, IList<T> values);

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Interfaces/ISearchLogic.cs

@@ -39 +39 @@
     bool IsMissingValue(int columnIndex, int rowIndex);
 
-    IEnumerable<T> GetValuesWithoutNaN<T>(int columnIndex);
+    IEnumerable<T> GetValuesWithoutNaN<T>(int columnIndex, bool considerSelection);
   }
 }

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Interfaces/IStatisticsLogic.cs

@@ -33 +33 @@
     int GetRowMissingValueCount(int rowIndex);
 
-    T GetMin<T>(int columnIndex) where T : IComparable<T>;
-    T GetMax<T>(int columnIndex) where T : IComparable<T>;
+    T GetMin<T>(int columnIndex, bool considerSelection) where T : IComparable<T>;
+    T GetMax<T>(int columnIndex, bool considerSelection) where T : IComparable<T>;
 
-    double GetMedian(int columnIndex);
-    double GetAverage(int columnIndex);
-    DateTime GetMedianDateTime(int columnIndex);
-    DateTime GetAverageDateTime(int columnIndex);
+    double GetMedian(int columnIndex, bool considerSelection);
+    double GetAverage(int columnIndex, bool considerSelection);
+    DateTime GetMedianDateTime(int columnIndex, bool considerSelection);
+    DateTime GetAverageDateTime(int columnIndex, bool considerSelection);
 
     double GetStandardDeviation(int columnIndex);
     double GetVariance(int columnIndex);
-    T GetMostCommonValue<T>(int columnIndex);
+    T GetMostCommonValue<T>(int columnIndex, bool considerSelection);
     int GetDifferentValuesCount<T>(int columnIndex);
 

branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/PreprocessingTransformator.cs

@@ -54 +54 @@
         int colIndex = preprocessingData.GetColumnIndex(transformation.Column);
 
-        var originalData = preprocessingData.GetValues<double>(colIndex);
+        var originalData = preprocessingData.GetValues<double>(colIndex, false);
         var transformedData = ApplyDoubleTransformation(transformation, originalData, out success);
         if (!success) return;