Changeset 15431 for branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content

Timestamp:

10/25/17 12:38:12 (7 years ago)

Author:

pfleck

Message:

#2809: Removed experimental static-typed datacolumns. (reverse merge g15291, r15309)

Location:

branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content

Files:

• DataGridContent.cs (modified) (6 diffs)
• ManipulationContent.cs (modified) (4 diffs)
• PreprocessingChartContent.cs (modified) (1 diff)
• ScatterPlotContent.cs (modified) (2 diffs)

branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/DataGridContent.cs

@@ -90 +90 @@
 
     public void DeleteRows(IEnumerable<int> rows) {
-      PreprocessingData.DeleteRows(rows);
+      PreprocessingData.DeleteRowsWithIndices(rows);
     }
 
@@ -134 +134 @@
 
     #region Manipulations
+    private void ReplaceIndicesByValue(IDictionary<int, IList<int>> cells, Func<int, double> doubleAggregator = null,
+      Func<int, DateTime> dateTimeAggregator = null, Func<int, string> stringAggregator = null) {
+      PreprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          if (doubleAggregator != null && PreprocessingData.VariableHasType<double>(column.Key)) {
+            var value = doubleAggregator(column.Key);
+            foreach (int index in column.Value)
+              PreprocessingData.SetCell<double>(column.Key, index, value);
+          } else if (dateTimeAggregator != null && PreprocessingData.VariableHasType<DateTime>(column.Key)) {
+            var value = dateTimeAggregator(column.Key);
+            foreach (int index in column.Value)
+              PreprocessingData.SetCell<DateTime>(column.Key, index, value);
+          } else if (stringAggregator != null && PreprocessingData.VariableHasType<string>(column.Key)) {
+            var value = stringAggregator(column.Key);
+            foreach (int index in column.Value)
+              PreprocessingData.SetCell<string>(column.Key, index, value);
+          }
+        }
+      });
+    }
+
+    private void ReplaceIndicesByValues(IDictionary<int, IList<int>> cells, Func<int, IEnumerable<double>> doubleAggregator = null,
+      Func<int, IEnumerable<DateTime>> dateTimeAggregator = null, Func<int, IEnumerable<string>> stringAggregator = null) {
+      PreprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          if (doubleAggregator != null && PreprocessingData.VariableHasType<double>(column.Key)) {
+            var values = doubleAggregator(column.Key);
+            foreach (var pair in column.Value.Zip(values, (row, value) => new { row, value }))
+              PreprocessingData.SetCell<double>(column.Key, pair.row, pair.value);
+          } else if (dateTimeAggregator != null && PreprocessingData.VariableHasType<DateTime>(column.Key)) {
+            var values = dateTimeAggregator(column.Key);
+            foreach (var pair in column.Value.Zip(values, (row, value) => new { row, value }))
+              PreprocessingData.SetCell<DateTime>(column.Key, pair.row, pair.value);
+          } else if (stringAggregator != null && PreprocessingData.VariableHasType<string>(column.Key)) {
+            var values = stringAggregator(column.Key);
+            foreach (var pair in column.Value.Zip(values, (row, value) => new { row, value }))
+              PreprocessingData.SetCell<string>(column.Key, pair.row, pair.value);
+          }
+        }
+      });
+    }
+
     public void ReplaceIndicesByMean(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
-      PreprocessingData.InTransaction(() => {
-        foreach (var column in cells) {
-          PreprocessingData.DataColumns[column.Key].TypeSwitch(
-            c => {
-              var mean = c.GetMean(considerSelection ? column.Value : null);
-              foreach (var index in column.Value) c[index] = mean;
-            },
-            dateTimeAction: c => {
-              var mean = c.GetMean(considerSelection ? column.Value : null);
-              foreach (var index in column.Value) c[index] = mean;
-            });
-        }
-      });
+      ReplaceIndicesByValue(cells,
+        col => PreprocessingData.GetMean<double>(col, considerSelection),
+        col => PreprocessingData.GetMean<DateTime>(col, considerSelection));
     }
 
     public void ReplaceIndicesByMedianValue(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
-      PreprocessingData.InTransaction(() => {
-        foreach (var column in cells) {
-          PreprocessingData.DataColumns[column.Key].TypeSwitch(
-            c => {
-              var median = c.GetMedian(considerSelection ? column.Value : null);
-              foreach (var index in column.Value) c[index] = median;
-            },
-            c => {
-              var median = c.GetMedian(considerSelection ? column.Value : null);
-              foreach (var index in column.Value) c[index] = median;
-            },
-            c => {
-              var median = c.GetMedian(considerSelection ? column.Value : null);
-              foreach (var index in column.Value) c[index] = median;
-            });
-        }
-      });
+      ReplaceIndicesByValue(cells,
+        col => PreprocessingData.GetMedian<double>(col, considerSelection),
+        col => PreprocessingData.GetMedian<DateTime>(col, considerSelection));
     }
 
     public void ReplaceIndicesByMode(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
-      PreprocessingData.InTransaction(() => {
-        foreach (var column in cells) {
-          PreprocessingData.DataColumns[column.Key].TypeSwitch(
-            c => {
-              var mode = c.GetMode(considerSelection ? column.Value : null);
-              foreach (var index in column.Value) c[index] = mode;
-            },
-            c => {
-              var mode = c.GetMode(considerSelection ? column.Value : null);
-              foreach (var index in column.Value) c[index] = mode;
-            },
-            c => {
-              var mode = c.GetMode(considerSelection ? column.Value : null);
-              foreach (var index in column.Value) c[index] = mode;
-            });
-        }
-      });
+      ReplaceIndicesByValue(cells,
+        col => PreprocessingData.GetMode<double>(col, considerSelection),
+        col => PreprocessingData.GetMode<DateTime>(col, considerSelection),
+        col => PreprocessingData.GetMode<string>(col, considerSelection));
     }
 
     public void ReplaceIndicesByRandomValue(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
       var rand = new FastRandom();
-      PreprocessingData.InTransaction(() => {
-        foreach (var column in cells) {
-          PreprocessingData.DataColumns[column.Key].TypeSwitch(
-            c => {
-              double min = c.GetMin(considerSelection ? column.Value : null);
-              double max = c.GetMax(considerSelection ? column.Value : null);
-              double range = max - min;
-              foreach (var index in column.Value) c[index] = min + rand.NextDouble() * range;
-            },
-            dateTimeAction: c => {
-              var min = c.GetMin(considerSelection ? column.Value : null);
-              var max = c.GetMax(considerSelection ? column.Value : null);
-              double range = (max - min).TotalSeconds;
-              foreach (var index in column.Value) c[index] = min + TimeSpan.FromSeconds(rand.NextDouble() * range);
-            });
-        }
-      });
+      ReplaceIndicesByValues(cells,
+        col => {
+          double min = PreprocessingData.GetMin<double>(col, considerSelection);
+          double max = PreprocessingData.GetMax<double>(col, considerSelection);
+          double range = max - min;
+          return cells[col].Select(_ => rand.NextDouble() * range + min);
+        },
+        col => {
+          var min = PreprocessingData.GetMin<DateTime>(col, considerSelection);
+          var max = PreprocessingData.GetMax<DateTime>(col, considerSelection);
+          double range = (max - min).TotalSeconds;
+          return cells[col].Select(_ => min + TimeSpan.FromSeconds(rand.NextDouble() * range));
+        });
     }
 
@@ -215 +216 @@
         foreach (var column in cells) {
           foreach (var rowIdx in column.Value) {
-            PreprocessingData.DataColumns[column.Key].SetValue(value, rowIdx);
+            PreprocessingData.SetValue(value, column.Key, rowIdx);
           }
         }
@@ -256 +257 @@
       int valuesToInterpolate = nextIndex - prevIndex;
 
-      PreprocessingData.DataColumns[column.Key].TypeSwitch(
-        c => {
-          double prev = c[prevIndex];
-          double next = c[nextIndex];
-          double interpolationStep = (next - prev) / valuesToInterpolate;
-          for (int i = prevIndex; i < nextIndex; i++) c[i] = prev + (interpolationStep * (i - prevIndex));
-        },
-        dateTimeAction: c => {
-          var prev = c[prevIndex];
-          var next = c[nextIndex];
-          double interpolationStep = (next - prev).TotalSeconds / valuesToInterpolate;
-          for (int i = prevIndex; i < nextIndex; i++) c[i] = prev.AddSeconds(interpolationStep * (i - prevIndex));
-        }
-      );
+      if (PreprocessingData.VariableHasType<double>(column.Key)) {
+        double prev = PreprocessingData.GetCell<double>(column.Key, prevIndex);
+        double next = PreprocessingData.GetCell<double>(column.Key, nextIndex);
+        double interpolationStep = (next - prev) / valuesToInterpolate;
+
+        for (int i = prevIndex; i < nextIndex; ++i) {
+          double interpolated = prev + (interpolationStep * (i - prevIndex));
+          PreprocessingData.SetCell<double>(column.Key, i, interpolated);
+        }
+      } else if (PreprocessingData.VariableHasType<DateTime>(column.Key)) {
+        DateTime prev = PreprocessingData.GetCell<DateTime>(column.Key, prevIndex);
+        DateTime next = PreprocessingData.GetCell<DateTime>(column.Key, nextIndex);
+        double interpolationStep = (next - prev).TotalSeconds / valuesToInterpolate;
+
+        for (int i = prevIndex; i < nextIndex; ++i) {
+          DateTime interpolated = prev.AddSeconds(interpolationStep * (i - prevIndex));
+          PreprocessingData.SetCell<DateTime>(column.Key, i, interpolated);
+        }
+      }
     }
 
     private int IndexOfPrevPresentValue(int columnIndex, int start) {
-      int index = start - 1;
-      while (index >= 0 && PreprocessingData.IsCellEmpty(columnIndex, index))
-        index--;
-      return index;
+      int offset = start - 1;
+      while (offset >= 0 && PreprocessingData.IsCellEmpty(columnIndex, offset)) {
+        offset--;
+      }
+
+      return offset;
     }
 
     private int IndexOfNextPresentValue(int columnIndex, int start) {
-      int index = start + 1;
-      while (index < PreprocessingData.Rows && PreprocessingData.IsCellEmpty(columnIndex, index))
-        index++;
-      return index;
+      int offset = start + 1;
+      while (offset < PreprocessingData.Rows && PreprocessingData.IsCellEmpty(columnIndex, offset)) {
+        offset++;
+      }
+
+      return offset;
     }
 
@@ -293 +303 @@
         PreprocessingData.InTransaction(() => {
           // process all given ranges - e.g. TrainingPartition, TestPartition
-          foreach (var range in ranges) {
+          foreach (IntRange range in ranges) {
             var indices = Enumerable.Range(0, PreprocessingData.Rows).ToArray();
             var shuffledIndices = Enumerable.Range(range.Start, range.Size).Shuffle(random).ToArray();
@@ -314 +324 @@
     public void ReOrderToIndices(int[] indices) {
       PreprocessingData.InTransaction(() => {
-        foreach (var column in PreprocessingData.DataColumns) {
-          column.TypeSwitch(
-            c => {
-              if (indices.Length != c.Values.Count) throw new InvalidOperationException("The number of provided indices does not match the values.");
-              var originalData = new List<double>(c.Values);
-              for (int i = 0; i < indices.Length; i++) c[i] = originalData[indices[i]];
-            },
-            c => {
-              if (indices.Length != c.Values.Count) throw new InvalidOperationException("The number of provided indices does not match the values.");
-              var originalData = new List<string>(c.Values);
-              for (int i = 0; i < indices.Length; i++) c[i] = originalData[indices[i]];
-            },
-            c => {
-              if (indices.Length != c.Values.Count) throw new InvalidOperationException("The number of provided indices does not match the values.");
-              var originalData = new List<DateTime>(c.Values);
-              for (int i = 0; i < indices.Length; i++) c[i] = originalData[indices[i]];
-            });
-        }
-      });
+        for (int i = 0; i < PreprocessingData.Columns; ++i) {
+          if (PreprocessingData.VariableHasType<double>(i))
+            ReOrderToIndices<double>(i, indices);
+          else if (PreprocessingData.VariableHasType<string>(i))
+            ReOrderToIndices<string>(i, indices);
+          else if (PreprocessingData.VariableHasType<DateTime>(i))
+            ReOrderToIndices<DateTime>(i, indices);
+        }
+      });
+    }
+
+    private void ReOrderToIndices<T>(int columnIndex, int[] indices) {
+      var originalData = new List<T>(PreprocessingData.GetValues<T>(columnIndex));
+      if (indices.Length != originalData.Count) throw new InvalidOperationException("The number of provided indices does not match the values.");
+
+      for (int i = 0; i < indices.Length; i++) {
+        T newValue = originalData[indices[i]];
+        PreprocessingData.SetCell<T>(columnIndex, i, newValue);
+      }
     }
     #endregion

branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/ManipulationContent.cs

@@ -57 +57 @@
 
       for (int i = 0; i < PreprocessingData.Rows; ++i) {
-        int missingCount = 0;
-        for (var col = 0; col < PreprocessingData.DataColumns.Count; col++) {
-          if (!PreprocessingData.DataColumns[col].IsValidValue(i))
-            missingCount++;
+        int missingCount = PreprocessingData.GetRowMissingValueCount(i);
+        if (100f / PreprocessingData.Columns * missingCount > percent) {
+          rows.Add(i);
         }
-        if (100f / PreprocessingData.Columns * missingCount > percent)
-          rows.Add(i);
       }
 
@@ -72 +69 @@
       List<int> columns = new List<int>();
       for (int i = 0; i < PreprocessingData.Columns; ++i) {
-        int missingCount = PreprocessingData.DataColumns[i].GetNumberOfMissingValues();
+        int missingCount = PreprocessingData.GetMissingValueCount(i);
         if (100f / PreprocessingData.Rows * missingCount > percent) {
           columns.Add(i);
@@ -83 +80 @@
     public List<int> ColumnsWithVarianceSmaller(double variance) {
       List<int> columns = new List<int>();
-
-      for (int i = 0; i < PreprocessingData.Columns; i++) {
-        if (PreprocessingData.DataColumns[i].TypeSwitch<bool>(
-          c => c.GetVariance() < variance,
-          c => false,
-          c => c.GetVariance().Ticks / TimeSpan.TicksPerSecond < variance
-        ))
-          columns.Add(i);
+      for (int i = 0; i < PreprocessingData.Columns; ++i) {
+        if (PreprocessingData.VariableHasType<double>(i)) {
+          double columnVariance = PreprocessingData.GetVariance<double>(i);
+          if (columnVariance < variance) {
+            columns.Add(i);
+          }
+        } else if (PreprocessingData.VariableHasType<DateTime>(i)) {
+          double columnVariance = (double)PreprocessingData.GetVariance<DateTime>(i).Ticks / TimeSpan.TicksPerSecond;
+          if (columnVariance < variance) {
+            columns.Add(i);
+          }
+        }
       }
-
       return columns;
     }
@@ -119 +119 @@
       PreprocessingData.InTransaction(() => {
         foreach (int column in columns.OrderByDescending(x => x)) {
-          PreprocessingData.DataColumns.RemoveAt(column);
+          PreprocessingData.DeleteColumn(column);
         }
       });

branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/PreprocessingChartContent.cs

@@ -82 +82 @@
 
     public static DataRow CreateDataRow(IFilteredPreprocessingData preprocessingData, string variableName, DataRowVisualProperties.DataRowChartType chartType) {
-      var values = preprocessingData.GetValues<double>(preprocessingData.GetColumnIndex(variableName));
+      IList<double> values = preprocessingData.GetValues<double>(preprocessingData.GetColumnIndex(variableName));
       DataRow row = new DataRow(variableName, "", values);
       row.VisualProperties.ChartType = chartType;

branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/ScatterPlotContent.cs

@@ -21 +21 @@
 
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using HeuristicLab.Analysis;
@@ -50 +51 @@
     #endregion
 
-    public static ScatterPlot CreateScatterPlot(IFilteredPreprocessingData preprocessingData, string variableNameX, string variableNameY,
-      string variableNameGroup = "-", LegendOrder legendOrder = LegendOrder.Alphabetically) {
+    public static ScatterPlot CreateScatterPlot(IFilteredPreprocessingData preprocessingData, string variableNameX, string variableNameY, string variableNameGroup = "-", LegendOrder legendOrder = LegendOrder.Alphabetically) {
       ScatterPlot scatterPlot = new ScatterPlot();
 
-      var xValues = preprocessingData.GetValues<double>(preprocessingData.GetColumnIndex(variableNameX));
-      var yValues = preprocessingData.GetValues<double>(preprocessingData.GetColumnIndex(variableNameY));
+      IList<double> xValues = preprocessingData.GetValues<double>(preprocessingData.GetColumnIndex(variableNameX));
+      IList<double> yValues = preprocessingData.GetValues<double>(preprocessingData.GetColumnIndex(variableNameY));
 
       var points = xValues.Zip(yValues, (x, y) => new Point2D<double>(x, y)).ToList();