Index: branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/DataGridContent.cs
===================================================================
--- branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/DataGridContent.cs	(revision 15291)
+++ branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/DataGridContent.cs	(revision 15309)
@@ -90,5 +90,5 @@
 
     public void DeleteRows(IEnumerable<int> rows) {
-      PreprocessingData.DeleteRowsWithIndices(rows);
+      PreprocessingData.DeleteRows(rows);
     }
 
@@ -134,80 +134,79 @@
 
     #region Manipulations
-    private void ReplaceIndicesByValue(IDictionary<int, IList<int>> cells, Func<int, double> doubleAggregator = null,
-      Func<int, DateTime> dateTimeAggregator = null, Func<int, string> stringAggregator = null) {
-      PreprocessingData.InTransaction(() => {
-        foreach (var column in cells) {
-          if (doubleAggregator != null && PreprocessingData.VariableHasType<double>(column.Key)) {
-            var value = doubleAggregator(column.Key);
-            foreach (int index in column.Value)
-              PreprocessingData.SetCell<double>(column.Key, index, value);
-          } else if (dateTimeAggregator != null && PreprocessingData.VariableHasType<DateTime>(column.Key)) {
-            var value = dateTimeAggregator(column.Key);
-            foreach (int index in column.Value)
-              PreprocessingData.SetCell<DateTime>(column.Key, index, value);
-          } else if (stringAggregator != null && PreprocessingData.VariableHasType<string>(column.Key)) {
-            var value = stringAggregator(column.Key);
-            foreach (int index in column.Value)
-              PreprocessingData.SetCell<string>(column.Key, index, value);
-          }
-        }
-      });
-    }
-
-    private void ReplaceIndicesByValues(IDictionary<int, IList<int>> cells, Func<int, IEnumerable<double>> doubleAggregator = null,
-      Func<int, IEnumerable<DateTime>> dateTimeAggregator = null, Func<int, IEnumerable<string>> stringAggregator = null) {
-      PreprocessingData.InTransaction(() => {
-        foreach (var column in cells) {
-          if (doubleAggregator != null && PreprocessingData.VariableHasType<double>(column.Key)) {
-            var values = doubleAggregator(column.Key);
-            foreach (var pair in column.Value.Zip(values, (row, value) => new { row, value }))
-              PreprocessingData.SetCell<double>(column.Key, pair.row, pair.value);
-          } else if (dateTimeAggregator != null && PreprocessingData.VariableHasType<DateTime>(column.Key)) {
-            var values = dateTimeAggregator(column.Key);
-            foreach (var pair in column.Value.Zip(values, (row, value) => new { row, value }))
-              PreprocessingData.SetCell<DateTime>(column.Key, pair.row, pair.value);
-          } else if (stringAggregator != null && PreprocessingData.VariableHasType<string>(column.Key)) {
-            var values = stringAggregator(column.Key);
-            foreach (var pair in column.Value.Zip(values, (row, value) => new { row, value }))
-              PreprocessingData.SetCell<string>(column.Key, pair.row, pair.value);
-          }
-        }
-      });
-    }
-
     public void ReplaceIndicesByMean(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
-      ReplaceIndicesByValue(cells,
-        col => PreprocessingData.GetMean<double>(col, considerSelection),
-        col => PreprocessingData.GetMean<DateTime>(col, considerSelection));
+      PreprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          PreprocessingData.DataColumns[column.Key].TypeSwitch(
+            c => {
+              var mean = c.GetMean(considerSelection ? column.Value : null);
+              foreach (var index in column.Value) c[index] = mean;
+            },
+            dateTimeAction: c => {
+              var mean = c.GetMean(considerSelection ? column.Value : null);
+              foreach (var index in column.Value) c[index] = mean;
+            });
+        }
+      });
     }
 
     public void ReplaceIndicesByMedianValue(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
-      ReplaceIndicesByValue(cells,
-        col => PreprocessingData.GetMedian<double>(col, considerSelection),
-        col => PreprocessingData.GetMedian<DateTime>(col, considerSelection));
+      PreprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          PreprocessingData.DataColumns[column.Key].TypeSwitch(
+            c => {
+              var median = c.GetMedian(considerSelection ? column.Value : null);
+              foreach (var index in column.Value) c[index] = median;
+            },
+            c => {
+              var median = c.GetMedian(considerSelection ? column.Value : null);
+              foreach (var index in column.Value) c[index] = median;
+            },
+            c => {
+              var median = c.GetMedian(considerSelection ? column.Value : null);
+              foreach (var index in column.Value) c[index] = median;
+            });
+        }
+      });
     }
 
     public void ReplaceIndicesByMode(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
-      ReplaceIndicesByValue(cells,
-        col => PreprocessingData.GetMode<double>(col, considerSelection),
-        col => PreprocessingData.GetMode<DateTime>(col, considerSelection),
-        col => PreprocessingData.GetMode<string>(col, considerSelection));
+      PreprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          PreprocessingData.DataColumns[column.Key].TypeSwitch(
+            c => {
+              var mode = c.GetMode(considerSelection ? column.Value : null);
+              foreach (var index in column.Value) c[index] = mode;
+            },
+            c => {
+              var mode = c.GetMode(considerSelection ? column.Value : null);
+              foreach (var index in column.Value) c[index] = mode;
+            },
+            c => {
+              var mode = c.GetMode(considerSelection ? column.Value : null);
+              foreach (var index in column.Value) c[index] = mode;
+            });
+        }
+      });
     }
 
     public void ReplaceIndicesByRandomValue(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
       var rand = new FastRandom();
-      ReplaceIndicesByValues(cells,
-        col => {
-          double min = PreprocessingData.GetMin<double>(col, considerSelection);
-          double max = PreprocessingData.GetMax<double>(col, considerSelection);
-          double range = max - min;
-          return cells[col].Select(_ => rand.NextDouble() * range + min);
-        },
-        col => {
-          var min = PreprocessingData.GetMin<DateTime>(col, considerSelection);
-          var max = PreprocessingData.GetMax<DateTime>(col, considerSelection);
-          double range = (max - min).TotalSeconds;
-          return cells[col].Select(_ => min + TimeSpan.FromSeconds(rand.NextDouble() * range));
-        });
+      PreprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          PreprocessingData.DataColumns[column.Key].TypeSwitch(
+            c => {
+              double min = c.GetMin(considerSelection ? column.Value : null);
+              double max = c.GetMax(considerSelection ? column.Value : null);
+              double range = max - min;
+              foreach (var index in column.Value) c[index] = min + rand.NextDouble() * range;
+            },
+            dateTimeAction: c => {
+              var min = c.GetMin(considerSelection ? column.Value : null);
+              var max = c.GetMax(considerSelection ? column.Value : null);
+              double range = (max - min).TotalSeconds;
+              foreach (var index in column.Value) c[index] = min + TimeSpan.FromSeconds(rand.NextDouble() * range);
+            });
+        }
+      });
     }
 
@@ -216,5 +215,5 @@
         foreach (var column in cells) {
           foreach (var rowIdx in column.Value) {
-            PreprocessingData.SetValue(value, column.Key, rowIdx);
+            PreprocessingData.DataColumns[column.Key].SetValue(value, rowIdx);
           }
         }
@@ -257,41 +256,32 @@
       int valuesToInterpolate = nextIndex - prevIndex;
 
-      if (PreprocessingData.VariableHasType<double>(column.Key)) {
-        double prev = PreprocessingData.GetCell<double>(column.Key, prevIndex);
-        double next = PreprocessingData.GetCell<double>(column.Key, nextIndex);
-        double interpolationStep = (next - prev) / valuesToInterpolate;
-
-        for (int i = prevIndex; i < nextIndex; ++i) {
-          double interpolated = prev + (interpolationStep * (i - prevIndex));
-          PreprocessingData.SetCell<double>(column.Key, i, interpolated);
-        }
-      } else if (PreprocessingData.VariableHasType<DateTime>(column.Key)) {
-        DateTime prev = PreprocessingData.GetCell<DateTime>(column.Key, prevIndex);
-        DateTime next = PreprocessingData.GetCell<DateTime>(column.Key, nextIndex);
-        double interpolationStep = (next - prev).TotalSeconds / valuesToInterpolate;
-
-        for (int i = prevIndex; i < nextIndex; ++i) {
-          DateTime interpolated = prev.AddSeconds(interpolationStep * (i - prevIndex));
-          PreprocessingData.SetCell<DateTime>(column.Key, i, interpolated);
-        }
-      }
+      PreprocessingData.DataColumns[column.Key].TypeSwitch(
+        c => {
+          double prev = c[prevIndex];
+          double next = c[nextIndex];
+          double interpolationStep = (next - prev) / valuesToInterpolate;
+          for (int i = prevIndex; i < nextIndex; i++) c[i] = prev + (interpolationStep * (i - prevIndex));
+        },
+        dateTimeAction: c => {
+          var prev = c[prevIndex];
+          var next = c[nextIndex];
+          double interpolationStep = (next - prev).TotalSeconds / valuesToInterpolate;
+          for (int i = prevIndex; i < nextIndex; i++) c[i] = prev.AddSeconds(interpolationStep * (i - prevIndex));
+        }
+      );
     }
 
     private int IndexOfPrevPresentValue(int columnIndex, int start) {
-      int offset = start - 1;
-      while (offset >= 0 && PreprocessingData.IsCellEmpty(columnIndex, offset)) {
-        offset--;
-      }
-
-      return offset;
+      int index = start - 1;
+      while (index >= 0 && PreprocessingData.IsCellEmpty(columnIndex, index))
+        index--;
+      return index;
     }
 
     private int IndexOfNextPresentValue(int columnIndex, int start) {
-      int offset = start + 1;
-      while (offset < PreprocessingData.Rows && PreprocessingData.IsCellEmpty(columnIndex, offset)) {
-        offset++;
-      }
-
-      return offset;
+      int index = start + 1;
+      while (index < PreprocessingData.Rows && PreprocessingData.IsCellEmpty(columnIndex, index))
+        index++;
+      return index;
     }
 
@@ -303,5 +293,5 @@
         PreprocessingData.InTransaction(() => {
           // process all given ranges - e.g. TrainingPartition, TestPartition
-          foreach (IntRange range in ranges) {
+          foreach (var range in ranges) {
             var indices = Enumerable.Range(0, PreprocessingData.Rows).ToArray();
             var shuffledIndices = Enumerable.Range(range.Start, range.Size).Shuffle(random).ToArray();
@@ -324,23 +314,23 @@
     public void ReOrderToIndices(int[] indices) {
       PreprocessingData.InTransaction(() => {
-        for (int i = 0; i < PreprocessingData.Columns; ++i) {
-          if (PreprocessingData.VariableHasType<double>(i))
-            ReOrderToIndices<double>(i, indices);
-          else if (PreprocessingData.VariableHasType<string>(i))
-            ReOrderToIndices<string>(i, indices);
-          else if (PreprocessingData.VariableHasType<DateTime>(i))
-            ReOrderToIndices<DateTime>(i, indices);
-        }
-      });
-    }
-
-    private void ReOrderToIndices<T>(int columnIndex, int[] indices) {
-      var originalData = new List<T>(PreprocessingData.GetValues<T>(columnIndex));
-      if (indices.Length != originalData.Count) throw new InvalidOperationException("The number of provided indices does not match the values.");
-
-      for (int i = 0; i < indices.Length; i++) {
-        T newValue = originalData[indices[i]];
-        PreprocessingData.SetCell<T>(columnIndex, i, newValue);
-      }
+        foreach (var column in PreprocessingData.DataColumns) {
+          column.TypeSwitch(
+            c => {
+              if (indices.Length != c.Values.Count) throw new InvalidOperationException("The number of provided indices does not match the values.");
+              var originalData = new List<double>(c.Values);
+              for (int i = 0; i < indices.Length; i++) c[i] = originalData[indices[i]];
+            },
+            c => {
+              if (indices.Length != c.Values.Count) throw new InvalidOperationException("The number of provided indices does not match the values.");
+              var originalData = new List<string>(c.Values);
+              for (int i = 0; i < indices.Length; i++) c[i] = originalData[indices[i]];
+            },
+            c => {
+              if (indices.Length != c.Values.Count) throw new InvalidOperationException("The number of provided indices does not match the values.");
+              var originalData = new List<DateTime>(c.Values);
+              for (int i = 0; i < indices.Length; i++) c[i] = originalData[indices[i]];
+            });
+        }
+      });
     }
     #endregion
Index: branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/ManipulationContent.cs
===================================================================
--- branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/ManipulationContent.cs	(revision 15291)
+++ branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/ManipulationContent.cs	(revision 15309)
@@ -57,8 +57,11 @@
 
       for (int i = 0; i < PreprocessingData.Rows; ++i) {
-        int missingCount = PreprocessingData.GetRowMissingValueCount(i);
-        if (100f / PreprocessingData.Columns * missingCount > percent) {
+        int missingCount = 0;
+        for (var col = 0; col < PreprocessingData.DataColumns.Count; col++) {
+          if (!PreprocessingData.DataColumns[col].IsValidValue(i))
+            missingCount++;
+        }
+        if (100f / PreprocessingData.Columns * missingCount > percent)
           rows.Add(i);
-        }
       }
 
@@ -69,5 +72,5 @@
       List<int> columns = new List<int>();
       for (int i = 0; i < PreprocessingData.Columns; ++i) {
-        int missingCount = PreprocessingData.GetMissingValueCount(i);
+        int missingCount = PreprocessingData.DataColumns[i].GetNumberOfMissingValues();
         if (100f / PreprocessingData.Rows * missingCount > percent) {
           columns.Add(i);
@@ -80,17 +83,14 @@
     public List<int> ColumnsWithVarianceSmaller(double variance) {
       List<int> columns = new List<int>();
-      for (int i = 0; i < PreprocessingData.Columns; ++i) {
-        if (PreprocessingData.VariableHasType<double>(i)) {
-          double columnVariance = PreprocessingData.GetVariance<double>(i);
-          if (columnVariance < variance) {
-            columns.Add(i);
-          }
-        } else if (PreprocessingData.VariableHasType<DateTime>(i)) {
-          double columnVariance = (double)PreprocessingData.GetVariance<DateTime>(i).Ticks / TimeSpan.TicksPerSecond;
-          if (columnVariance < variance) {
-            columns.Add(i);
-          }
-        }
+
+      for (int i = 0; i < PreprocessingData.Columns; i++) {
+        if (PreprocessingData.DataColumns[i].TypeSwitch<bool>(
+          c => c.GetVariance() < variance,
+          c => false,
+          c => c.GetVariance().Ticks / TimeSpan.TicksPerSecond < variance
+        ))
+          columns.Add(i);
       }
+
       return columns;
     }
@@ -119,5 +119,5 @@
       PreprocessingData.InTransaction(() => {
         foreach (int column in columns.OrderByDescending(x => x)) {
-          PreprocessingData.DeleteColumn(column);
+          PreprocessingData.DataColumns.RemoveAt(column);
         }
       });
Index: branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/PreprocessingChartContent.cs
===================================================================
--- branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/PreprocessingChartContent.cs	(revision 15291)
+++ branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/PreprocessingChartContent.cs	(revision 15309)
@@ -82,5 +82,5 @@
 
     public static DataRow CreateDataRow(IFilteredPreprocessingData preprocessingData, string variableName, DataRowVisualProperties.DataRowChartType chartType) {
-      IList<double> values = preprocessingData.GetValues<double>(preprocessingData.GetColumnIndex(variableName));
+      var values = preprocessingData.GetValues<double>(preprocessingData.GetColumnIndex(variableName));
       DataRow row = new DataRow(variableName, "", values);
       row.VisualProperties.ChartType = chartType;
Index: branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/ScatterPlotContent.cs
===================================================================
--- branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/ScatterPlotContent.cs	(revision 15291)
+++ branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/ScatterPlotContent.cs	(revision 15309)
@@ -21,5 +21,4 @@
 
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using HeuristicLab.Analysis;
@@ -51,9 +50,10 @@
     #endregion
 
-    public static ScatterPlot CreateScatterPlot(IFilteredPreprocessingData preprocessingData, string variableNameX, string variableNameY, string variableNameGroup = "-", LegendOrder legendOrder = LegendOrder.Alphabetically) {
+    public static ScatterPlot CreateScatterPlot(IFilteredPreprocessingData preprocessingData, string variableNameX, string variableNameY,
+      string variableNameGroup = "-", LegendOrder legendOrder = LegendOrder.Alphabetically) {
       ScatterPlot scatterPlot = new ScatterPlot();
 
-      IList<double> xValues = preprocessingData.GetValues<double>(preprocessingData.GetColumnIndex(variableNameX));
-      IList<double> yValues = preprocessingData.GetValues<double>(preprocessingData.GetColumnIndex(variableNameY));
+      var xValues = preprocessingData.GetValues<double>(preprocessingData.GetColumnIndex(variableNameX));
+      var yValues = preprocessingData.GetValues<double>(preprocessingData.GetColumnIndex(variableNameY));
 
       var points = xValues.Zip(yValues, (x, y) => new Point2D<double>(x, y)).ToList();
Index: branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/Columns/DateTimePreprocessingDataColumn.cs
===================================================================
--- branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/Columns/DateTimePreprocessingDataColumn.cs	(revision 15291)
+++ branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/Columns/DateTimePreprocessingDataColumn.cs	(revision 15309)
@@ -29,5 +29,5 @@
 namespace HeuristicLab.DataPreprocessing {
   [Item("DateTimePreprocessingDataColumn", "")]
-  public class DateTimePreprocessingDataColumn : NullablePreprocessingDataColumn<DateTime, TimeSpan> {
+  public class DateTimePreprocessingDataColumn : PreprocessingDataColumn<DateTime> {
 
     #region Constructor, Cloning & Persistence
@@ -35,7 +35,4 @@
       : base() { }
     public DateTimePreprocessingDataColumn(string name, IEnumerable<DateTime> values)
-      : base(name, values) {
-    }
-    public DateTimePreprocessingDataColumn(string name, IEnumerable<DateTime?> values)
       : base(name, values) {
     }
@@ -53,9 +50,11 @@
     #endregion
 
+    protected override DateTime DefaultValue { get { return DateTime.MinValue; } }
+
     #region Statistics
-    public override TimeSpan GetRange() { return ValidValues.Max() - ValidValues.Min(); }
-    public override DateTime GetMean() { return AggregateAsDouble(ValidValues, Enumerable.Average); }
-    public override TimeSpan GetStandardDeviation() { return AggregateDistanceAsDouble(ValidValues, EnumerableStatisticExtensions.StandardDeviation); }
-    public override TimeSpan GetVariance() { return AggregateDistanceAsDouble(ValidValues, EnumerableStatisticExtensions.Variance); }
+    public TimeSpan GetRange(IEnumerable<int> indices = null) { return GetMax(indices) - GetMin(indices); }
+    public DateTime GetMean(IEnumerable<int> indices = null) { return AggregateAsDouble(GetValidValues(indices), Enumerable.Average); }
+    public TimeSpan GetStandardDeviation(IEnumerable<int> indices = null) { return AggregateDistanceAsDouble(GetValidValues(indices), EnumerableStatisticExtensions.StandardDeviation); }
+    public TimeSpan GetVariance(IEnumerable<int> indices = null) { return AggregateDistanceAsDouble(GetValidValues(indices), EnumerableStatisticExtensions.Variance); }
     #endregion
 
@@ -69,5 +68,5 @@
     public override string GetValue(int index) {
       var value = Values[index];
-      return value.HasValue ? value.Value.ToString("o") : string.Empty;
+      return IsValidValue(value) ? Values[index].ToString("o") : string.Empty;   // format "s" sortable or "o" roundtrip
     }
     public override bool SetValue(string value, int index) {
@@ -77,5 +76,5 @@
         return true;
       } else if (string.IsNullOrEmpty(value)) {
-        Values[index] = null;
+        Values[index] = DateTime.MinValue;
         return true;
       } else {
@@ -86,8 +85,8 @@
 
     private static DateTime AggregateAsDouble(IEnumerable<DateTime> values, Func<IEnumerable<double>, double> func) {
-      return new DateTime((long)func(values.Select(x => (double)x.Ticks)));
+      return values.Any() ? new DateTime((long)func(values.Select(x => (double)x.Ticks))) : DateTime.MinValue;
     }
     private static TimeSpan AggregateDistanceAsDouble(IEnumerable<DateTime> values, Func<IEnumerable<double>, double> func) {
-      return new TimeSpan((long)func(values.Select(x => (double)x.Ticks)));
+      return values.Any() ? new TimeSpan((long)func(values.Select(x => (double)x.Ticks))) : TimeSpan.Zero;
     }
   }
Index: branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/Columns/DoublePreprocessingDataColumn.cs
===================================================================
--- branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/Columns/DoublePreprocessingDataColumn.cs	(revision 15291)
+++ branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/Columns/DoublePreprocessingDataColumn.cs	(revision 15309)
@@ -31,19 +31,17 @@
   [Item("DoublePreprocessingDataColumn", "")]
   [StorableClass]
-  public sealed class DoublePreprocessingDataColumn : NullablePreprocessingDataColumn<double, double> {
+  public sealed class DoublePreprocessingDataColumn : PreprocessingDataColumn<double> {
 
     #region Constructor, Cloning & Persistence
+
     public DoublePreprocessingDataColumn()
       : base() { }
+
     public DoublePreprocessingDataColumn(string name, IEnumerable<double> values)
-      : base(name, values) {
-    }
-    public DoublePreprocessingDataColumn(string name, IEnumerable<double?> values)
-      : base(name, values) {
-    }
+      : base(name, values) { }
 
     private DoublePreprocessingDataColumn(DoublePreprocessingDataColumn original, Cloner cloner)
-      : base(original, cloner) {
-    }
+      : base(original, cloner) { }
+
     public override IDeepCloneable Clone(Cloner cloner) {
       return new DoublePreprocessingDataColumn(this, cloner);
@@ -53,4 +51,5 @@
     private DoublePreprocessingDataColumn(bool deserializing)
       : base(deserializing) { }
+
     #endregion
 
@@ -59,11 +58,13 @@
     }
 
+    protected override double DefaultValue { get { return double.NaN; } }
+
     #region Statistics
-    public override double GetRange() { return ValidValues.Max() - ValidValues.Min(); }
-    public override double GetMean() { return ValidValues.Average(); }
-    public override double GetMedian() { return ValidValues.Quantile(0.5); } // IEnumerable<doube> version is faster
-    public override double GetStandardDeviation() { return ValidValues.StandardDeviation(); }
-    public override double GetVariance() { return ValidValues.Variance(); }
-    public override double GetQuantile(double alpha) { return ValidValues.Quantile(alpha); } // IEnumerable<doube> version is faster
+    public double GetRange(IEnumerable<int> indices = null) { return GetMax(indices) - GetMin(indices); }
+    public double GetMean(IEnumerable<int> indices = null) { return GetValidValues(indices).DefaultIfEmpty(DefaultValue).Average(); }
+    public override double GetMedian(IEnumerable<int> indices = null) { return GetValidValues(indices).DefaultIfEmpty(DefaultValue).Median(); } // IEnumerable<doube> version is faster
+    public double GetStandardDeviation(IEnumerable<int> indices = null) { return GetValidValues(indices).DefaultIfEmpty(DefaultValue).StandardDeviation(); }
+    public double GetVariance(IEnumerable<int> indices = null) { return GetValidValues(indices).DefaultIfEmpty(DefaultValue).Variance(); }
+    public override double GetQuantile(double alpha, IEnumerable<int> indices = null) { return GetValidValues(indices).DefaultIfEmpty(double.NaN).Quantile(alpha); } // IEnumerable<doube> version is faster
     #endregion
 
@@ -83,6 +84,5 @@
     }
     public override string GetValue(int index) {
-      var value = Values[index];
-      return value.HasValue ? value.Value.ToString("r") : string.Empty;
+      return Values[index].ToString("r");
     }
     public override bool SetValue(string value, int index) {
@@ -92,5 +92,5 @@
         return true;
       } else if (string.IsNullOrEmpty(value)) {
-        Values[index] = null;
+        Values[index] = double.NaN;
         return true;
       } else {
Index: branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/Columns/PreprocessingDataColumn.cs
===================================================================
--- branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/Columns/PreprocessingDataColumn.cs	(revision 15291)
+++ branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/Columns/PreprocessingDataColumn.cs	(revision 15309)
@@ -54,4 +54,75 @@
     public abstract bool IsValidValue(int index);
 
+    #region Column Type Switches
+    internal void TypeSwitch(Action<DoublePreprocessingDataColumn> doubleAction, Action<StringPreprocessingDataColumn> stringAction = null, Action<DateTimePreprocessingDataColumn> dateTimeAction = null) {
+      var doubleColumn = this as DoublePreprocessingDataColumn;
+      if (doubleColumn != null && doubleAction != null) doubleAction(doubleColumn);
+      var stringColumn = this as StringPreprocessingDataColumn;
+      if (stringColumn != null && stringAction != null) stringAction(stringColumn);
+      var dateTimeColumn = this as DateTimePreprocessingDataColumn;
+      if (dateTimeColumn != null && dateTimeAction != null) dateTimeAction(dateTimeColumn);
+    }
+    internal void TypeSwitch<TIn>(TIn value, Action<DoublePreprocessingDataColumn, double> doubleAction, Action<StringPreprocessingDataColumn, string> stringAction = null, Action<DateTimePreprocessingDataColumn, DateTime> dateTimeAction = null) {
+      var doubleColumn = this as DoublePreprocessingDataColumn;
+      if (doubleColumn != null && doubleAction != null) doubleAction(doubleColumn, Convert<double>(value));
+      var stringColumn = this as StringPreprocessingDataColumn;
+      if (stringColumn != null && stringAction != null) stringAction(stringColumn, Convert<string>(value));
+      var dateTimeColumn = this as DateTimePreprocessingDataColumn;
+      if (dateTimeColumn != null && dateTimeAction != null) dateTimeAction(dateTimeColumn, Convert<DateTime>(value));
+    }
+
+    internal TOut TypeSwitch<TOut>(Func<DoublePreprocessingDataColumn, double> doubleFunc, Func<StringPreprocessingDataColumn, string> stringFunc = null, Func<DateTimePreprocessingDataColumn, DateTime> dateTimeFunc = null) {
+      var doubleColumn = this as DoublePreprocessingDataColumn;
+      if (doubleColumn != null && doubleFunc != null) return Convert<TOut>(doubleFunc(doubleColumn));
+      var stringColumn = this as StringPreprocessingDataColumn;
+      if (stringColumn != null && stringFunc != null) return Convert<TOut>(stringFunc(stringColumn));
+      var dateTimeColumn = this as DateTimePreprocessingDataColumn;
+      if (dateTimeColumn != null && dateTimeFunc != null) return Convert<TOut>(dateTimeFunc(dateTimeColumn));
+      throw new InvalidOperationException("Invalid data column type.");
+    }
+    internal TOut TypeSwitch<TOut>(Func<DoublePreprocessingDataColumn, TOut> doubleFunc, Func<StringPreprocessingDataColumn, TOut> stringFunc = null, Func<DateTimePreprocessingDataColumn, TOut> dateTimeFunc = null) {
+      var doubleColumn = this as DoublePreprocessingDataColumn;
+      if (doubleColumn != null && doubleFunc != null) return doubleFunc(doubleColumn);
+      var stringColumn = this as StringPreprocessingDataColumn;
+      if (stringColumn != null && stringFunc != null) return stringFunc(stringColumn);
+      var dateTimeColumn = this as DateTimePreprocessingDataColumn;
+      if (dateTimeColumn != null && dateTimeFunc != null) return dateTimeFunc(dateTimeColumn);
+      throw new InvalidOperationException("Invalid data column type.");
+    }
+    internal TOut TypeSwitch<TIn, TOut>(TIn value, Func<DoublePreprocessingDataColumn, double, TOut> doubleFunc, Func<StringPreprocessingDataColumn, string, TOut> stringFunc = null, Func<DateTimePreprocessingDataColumn, DateTime, TOut> dateTimeFunc = null) {
+      var doubleColumn = this as DoublePreprocessingDataColumn;
+      if (doubleColumn != null && doubleFunc != null) return doubleFunc(doubleColumn, Convert<double>(value));
+      var stringColumn = this as StringPreprocessingDataColumn;
+      if (stringColumn != null && stringFunc != null) return stringFunc(stringColumn, Convert<string>(value));
+      var dateTimeColumn = this as DateTimePreprocessingDataColumn;
+      if (dateTimeColumn != null && dateTimeFunc != null) return dateTimeFunc(dateTimeColumn, Convert<DateTime>(value));
+      throw new InvalidOperationException("Invalid data column type.");
+    }
+    internal IEnumerable<TOut> TypeSwitch<TOut>(Func<DoublePreprocessingDataColumn, IEnumerable<double>> doubleFunc, Func<StringPreprocessingDataColumn, IEnumerable<string>> stringFunc = null, Func<DateTimePreprocessingDataColumn, IEnumerable<DateTime>> dateTimeFunc = null) {
+      var doubleColumn = this as DoublePreprocessingDataColumn;
+      if (doubleColumn != null && doubleFunc != null) return Convert<IEnumerable<TOut>>(doubleFunc(doubleColumn));
+      var stringColumn = this as StringPreprocessingDataColumn;
+      if (stringColumn != null && stringFunc != null) return Convert<IEnumerable<TOut>>(stringFunc(stringColumn));
+      var dateTimeColumn = this as DateTimePreprocessingDataColumn;
+      if (dateTimeColumn != null && dateTimeFunc != null) return Convert<IEnumerable<TOut>>(dateTimeFunc(dateTimeColumn));
+      throw new InvalidOperationException("Invalid data column type.");
+    }
+    internal IEnumerable<TOut> TypeSwitch<TOut, TIn>(TIn value, Func<DoublePreprocessingDataColumn, double, IEnumerable<double>> doubleFunc, Func<StringPreprocessingDataColumn, string, IEnumerable<string>> stringFunc = null, Func<DateTimePreprocessingDataColumn, DateTime, IEnumerable<DateTime>> dateTimeFunc = null) {
+      var doubleColumn = this as DoublePreprocessingDataColumn;
+      if (doubleColumn != null && doubleFunc != null) return Convert<IEnumerable<TOut>>(doubleFunc(doubleColumn, Convert<double>(value)));
+      var stringColumn = this as StringPreprocessingDataColumn;
+      if (stringColumn != null && stringFunc != null) return Convert<IEnumerable<TOut>>(stringFunc(stringColumn, Convert<string>(value)));
+      var dateTimeColumn = this as DateTimePreprocessingDataColumn;
+      if (dateTimeColumn != null && dateTimeFunc != null) return Convert<IEnumerable<TOut>>(dateTimeFunc(dateTimeColumn, Convert<DateTime>(value)));
+      throw new InvalidOperationException("Invalid data column type.");
+    }
+
+    private static T Convert<T>(object obj) { return (T)obj; }
+    #endregion
+
+    #region Statistics
+    public abstract int GetDistinctValues(IEnumerable<int> indices = null);
+    public abstract int GetNumberOfMissingValues(IEnumerable<int> indices = null);
+    #endregion
 
     #region String Handling
@@ -64,18 +135,18 @@
   [Item("PreprocessingDataColumn", "")]
   [StorableClass]
-  public abstract class PreprocessingDataColumn<TValue, TDistance> : PreprocessingDataColumn
-    where TValue : class, IComparable<TValue> {
+  public abstract class PreprocessingDataColumn<T> : PreprocessingDataColumn
+    where T : IComparable<T> {
 
     #region Constructor, Cloning & Persistence
     protected PreprocessingDataColumn()
-      : this(string.Empty, Enumerable.Empty<TValue>()) { }
-    protected PreprocessingDataColumn(string name, IEnumerable<TValue> values)
+      : this(string.Empty, Enumerable.Empty<T>()) { }
+    protected PreprocessingDataColumn(string name, IEnumerable<T> values)
       : base(name) {
-      Values = new List<TValue>(values);
-    }
-
-    protected PreprocessingDataColumn(PreprocessingDataColumn<TValue, TDistance> original, Cloner cloner)
+      Values = new List<T>(values);
+    }
+
+    protected PreprocessingDataColumn(PreprocessingDataColumn<T> original, Cloner cloner)
       : base(original, cloner) {
-      Values = new List<TValue>(original.Values);
+      Values = new List<T>(original.Values);
     }
 
@@ -86,11 +157,20 @@
 
     [Storable]
-    public List<TValue> Values { get; private set; }
-    public IEnumerable<TValue> ValidValues {
-      get { return Values.Where(IsValidValue); }
-    }
+    internal List<T> Values { get; private set; }
+    public IEnumerable<T> GetValues(IEnumerable<int> indices = null) {
+      return indices == null
+        ? Values
+        : indices.Select(index => Values[index]);
+    }
+    public IEnumerable<T> GetValidValues(IEnumerable<int> indices = null) {
+      return indices == null
+        ? Values.Where(IsValidValue)
+        : indices.Select(index => Values[index]).Where(IsValidValue);
+    }
+
+    protected abstract T DefaultValue { get; }
 
     public override Type GetValueType() {
-      return typeof(TValue);
+      return typeof(T);
     }
 
@@ -99,10 +179,10 @@
     }
 
-    public TValue this[int index] {
+    public T this[int index] {
       get { return Values[index]; }
       set { Values[index] = value; }
     }
 
-    public virtual bool IsValidValue(TValue value) { return true; }
+    public virtual bool IsValidValue(T value) { return true; }
     public override bool IsValidValue(int index) {
       return IsValidValue(Values[index]);
@@ -110,80 +190,12 @@
 
     #region Statistics
-    public virtual TValue GetMin() { return Values.Min(); }
-    public virtual TValue GetMax() { return Values.Max(); }
-    public abstract TDistance GetRange();
-    public abstract TValue GetMean();
-    public virtual TValue GetMedian() { return Values.Quantile(0.5); }
-    public virtual TValue GetMode() { return Values.GroupBy(x => x).OrderByDescending(g => g.Count()).Select(g => g.Key).First(); }
-    public abstract TDistance GetStandardDeviation();
-    public abstract TDistance GetVariance();
-    public virtual TValue GetQuantile(double alpha) { return Values.Quantile(alpha); }
-    public virtual int GetDistinctValues() { return Values.GroupBy(x => x).Count(); }
-    public virtual int GetNumberOfMissingValues() { return Values.Count(IsValidValue); }
-    #endregion
-  }
-
-  [Item("NullablePreprocessingDataColumn", "")]
-  [StorableClass]
-  public abstract class NullablePreprocessingDataColumn<TValue, TDistance> : PreprocessingDataColumn
-    where TValue : struct, IComparable<TValue> {
-
-    #region Constructor, Cloning & Persistence
-    protected NullablePreprocessingDataColumn()
-      : this(string.Empty, Enumerable.Empty<TValue?>()) { }
-    protected NullablePreprocessingDataColumn(string name, IEnumerable<TValue> values)
-      : this(name, values.Select(x => (TValue?)x)) { }
-    protected NullablePreprocessingDataColumn(string name, IEnumerable<TValue?> values)
-      : base(name) {
-      Values = new List<TValue?>(values);
-    }
-
-    protected NullablePreprocessingDataColumn(NullablePreprocessingDataColumn<TValue, TDistance> original, Cloner cloner)
-      : base(original, cloner) {
-      Values = new List<TValue?>(original.Values);
-    }
-
-    [StorableConstructor]
-    protected NullablePreprocessingDataColumn(bool deserializing)
-      : base(deserializing) { }
-    #endregion
-
-    [Storable]
-    internal List<TValue?> Values { get; private set; }
-    protected IEnumerable<TValue> ValidValues {
-      get { return Values.Where(x => x.HasValue && IsValidValue(x.Value)).Select(x => x.Value); }
-    }
-
-    public override Type GetValueType() {
-      return typeof(TValue);
-    }
-
-    public override int Length {
-      get { return Values.Count; }
-    }
-
-    public TValue? this[int index] {
-      get { return Values[index]; }
-      set { Values[index] = value; }
-    }
-
-    public virtual bool IsValidValue(TValue value) { return true; }
-    public override bool IsValidValue(int index) {
-      var value = Values[index];
-      return value.HasValue && IsValidValue(value.Value);
-    }
-
-    #region Statistics
-    public virtual TValue GetMin() { return ValidValues.Min(); }
-    public virtual TValue GetMax() { return ValidValues.Max(); }
-    public abstract TDistance GetRange();
-    public abstract TValue GetMean();
-    public virtual TValue GetMedian() { return ValidValues.Quantile(0.5); }
-    public virtual TValue GetMode() { return ValidValues.GroupBy(x => x).OrderByDescending(g => g.Count()).Select(g => g.Key).First(); }
-    public abstract TDistance GetStandardDeviation();
-    public abstract TDistance GetVariance();
-    public virtual TValue GetQuantile(double alpha) { return ValidValues.Quantile(alpha); }
-    public virtual int GetDistinctValues() { return ValidValues.GroupBy(x => x).Count(); }
-    public virtual int GetNumberOfMissingValues() { return Values.Count - ValidValues.Count(); }
+
+    public virtual T GetMin(IEnumerable<int> indices = null) { return GetValidValues(indices).DefaultIfEmpty(DefaultValue).Min(); }
+    public virtual T GetMax(IEnumerable<int> indices = null) { return GetValidValues(indices).DefaultIfEmpty(DefaultValue).Max(); }
+    public virtual T GetMedian(IEnumerable<int> indices = null) { return GetValidValues(indices).DefaultIfEmpty(DefaultValue).Quantile(0.5); }
+    public virtual T GetMode(IEnumerable<int> indices = null) { return GetValidValues(indices).DefaultIfEmpty(DefaultValue).GroupBy(x => x).OrderByDescending(g => g.Count()).Select(g => g.Key).First(); }
+    public virtual T GetQuantile(double alpha, IEnumerable<int> indices = null) { return GetValidValues(indices).DefaultIfEmpty(DefaultValue).Quantile(alpha); }
+    public override int GetDistinctValues(IEnumerable<int> indices = null) { return GetValidValues(indices).GroupBy(x => x).Count(); }
+    public override int GetNumberOfMissingValues(IEnumerable<int> indices = null) { return GetValues(indices).Count(x => !IsValidValue(x)); }
     #endregion
   }
Index: branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/Columns/StringPreprocessingDataColumn.cs
===================================================================
--- branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/Columns/StringPreprocessingDataColumn.cs	(revision 15291)
+++ branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/Columns/StringPreprocessingDataColumn.cs	(revision 15309)
@@ -28,5 +28,5 @@
   [Item("StringPreprocessingDataColumn", "")]
   [StorableClass]
-  public sealed class StringPreprocessingDataColumn : PreprocessingDataColumn<string, string> {
+  public sealed class StringPreprocessingDataColumn : PreprocessingDataColumn<string> {
 
     #region Constructor, Cloning & Persistence
@@ -53,20 +53,10 @@
     }
 
-    #region Statistics
-    public override string GetRange() { return string.Empty; }
-    public override string GetMean() { return string.Empty; }
-    public override string GetStandardDeviation() { return string.Empty; }
-    public override string GetVariance() { return string.Empty; }
-    #endregion
+    protected override string DefaultValue { get { return string.Empty; } }
 
     #region IStringConvertibleColumn
     public override bool Validate(string value, out string errorMessage) {
-      if (value == null) {
-        errorMessage = "Invalid Value (string must not be null)";
-        return false;
-      } else {
-        errorMessage = string.Empty;
-        return true;
-      }
+      errorMessage = string.Empty;
+      return true;
     }
     public override string GetValue(int index) {
Index: branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/FilteredPreprocessingData.cs
===================================================================
--- branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/FilteredPreprocessingData.cs	(revision 15291)
+++ branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/FilteredPreprocessingData.cs	(revision 15309)
@@ -38,4 +38,8 @@
     private IPreprocessingData filteredData;
 
+    public IList<PreprocessingDataColumn> DataColumns {
+      get { return ActiveData.DataColumns; }
+    }
+
     public IPreprocessingData ActiveData {
       get { return IsFiltered ? filteredData : originalData; }
@@ -82,9 +86,9 @@
     }
 
-    public IList<T> GetValues<T>(int columnIndex, bool considerSelection) {
+    public IEnumerable<T> GetValues<T>(int columnIndex, bool considerSelection) {
       return ActiveData.GetValues<T>(columnIndex, considerSelection);
     }
 
-    public void SetValues<T>(int columnIndex, IList<T> values) {
+    public void SetValues<T>(int columnIndex, IEnumerable<T> values) {
       if (IsFiltered)
         throw new InvalidOperationException("SetValues not possible while data is filtered");
@@ -123,9 +127,9 @@
     }
 
-    public void DeleteRowsWithIndices(IEnumerable<int> rows) {
+    public void DeleteRows(IEnumerable<int> rows) {
       if (IsFiltered)
         throw new InvalidOperationException("DeleteRowsWithIndices not possible while data is filtered");
 
-      originalData.DeleteRowsWithIndices(rows);
+      originalData.DeleteRows(rows);
     }
 
@@ -273,44 +277,4 @@
     public void EndTransaction() {
       originalData.EndTransaction();
-    }
-    #endregion
-
-    #region Statistics
-    public T GetMin<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
-      return ActiveData.GetMin<T>(columnIndex, considerSelection, emptyValue);
-    }
-    public T GetMax<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
-      return ActiveData.GetMax<T>(columnIndex, considerSelection, emptyValue);
-    }
-    public T GetMean<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
-      return ActiveData.GetMean<T>(columnIndex, considerSelection, emptyValue);
-    }
-    public T GetMedian<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IComparable<T> {
-      return ActiveData.GetMean<T>(columnIndex, considerSelection, emptyValue);
-    }
-    public T GetMode<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IEquatable<T> {
-      return ActiveData.GetMode<T>(columnIndex, considerSelection, emptyValue);
-    }
-    public T GetStandardDeviation<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
-      return ActiveData.GetStandardDeviation<T>(columnIndex, considerSelection, emptyValue);
-    }
-    public T GetVariance<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
-      return ActiveData.GetVariance<T>(columnIndex, considerSelection, emptyValue);
-    }
-    public T GetQuantile<T>(double alpha, int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IComparable<T> {
-      return ActiveData.GetQuantile<T>(alpha, columnIndex, considerSelection, emptyValue);
-    }
-    public int GetDistinctValues<T>(int columnIndex, bool considerSelection = false) {
-      return ActiveData.GetDistinctValues<T>(columnIndex, considerSelection);
-    }
-
-    public int GetMissingValueCount() {
-      return ActiveData.GetMissingValueCount();
-    }
-    public int GetMissingValueCount(int columnIndex) {
-      return ActiveData.GetMissingValueCount(columnIndex);
-    }
-    public int GetRowMissingValueCount(int rowIndex) {
-      return ActiveData.GetRowMissingValueCount(rowIndex);
     }
     #endregion
Index: branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/IPreprocessingData.cs
===================================================================
--- branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/IPreprocessingData.cs	(revision 15291)
+++ branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/IPreprocessingData.cs	(revision 15309)
@@ -28,4 +28,7 @@
 namespace HeuristicLab.DataPreprocessing {
   public interface IPreprocessingData : INamedItem {
+
+    IList<PreprocessingDataColumn> DataColumns { get; }
+
     #region Cells
     bool IsCellEmpty(int columnIndex, int rowIndex);
@@ -36,7 +39,7 @@
     string GetCellAsString(int columnIndex, int rowIndex);
 
-    IList<T> GetValues<T>(int columnIndex, bool considerSelection = false);
+    IEnumerable<T> GetValues<T>(int columnIndex, bool considerSelection = false);
 
-    void SetValues<T>(int columnIndex, IList<T> values);
+    void SetValues<T>(int columnIndex, IEnumerable<T> values);
     bool SetValue(string value, int columnIndex, int rowIndex);
 
@@ -48,5 +51,5 @@
     void InsertRow(int rowIndex);
     void DeleteRow(int rowIndex);
-    void DeleteRowsWithIndices(IEnumerable<int> rows);
+    void DeleteRows(IEnumerable<int> rows);
     void InsertColumn<T>(string variableName, int columnIndex);
 
@@ -106,20 +109,4 @@
     void EndTransaction();
     #endregion
-
-    #region Statistics
-    T GetMin<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T));
-    T GetMax<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T));
-    T GetMean<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T));
-    T GetMedian<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IComparable<T>;
-    T GetMode<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IEquatable<T>;
-    T GetStandardDeviation<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T));
-    T GetVariance<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T));
-    T GetQuantile<T>(double alpha, int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IComparable<T>;
-    int GetDistinctValues<T>(int columnIndex, bool considerSelection = false);
-
-    int GetMissingValueCount();
-    int GetMissingValueCount(int columnIndex);
-    int GetRowMissingValueCount(int rowIndex);
-    #endregion
   }
 }
Index: branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/PreprocessingData.cs
===================================================================
--- branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/PreprocessingData.cs	(revision 15291)
+++ branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/PreprocessingData.cs	(revision 15309)
@@ -36,6 +36,10 @@
   public class PreprocessingData : NamedItem, IPreprocessingData {
 
-    [Storable]
-    protected List<PreprocessingDataColumn> dataColumns;
+    [Storable] private List<PreprocessingDataColumn> dataColumns;
+
+    public IList<PreprocessingDataColumn> DataColumns {
+      get { return dataColumns; }
+    }
+
 
     #region Constructor, Cloning & Persistence
@@ -99,80 +103,6 @@
     }
 
-    private void ColumnTypeSwitchAction<T>(int columnIndex, T value, Action<DoublePreprocessingDataColumn, double?> doubleAction,
-      Action<StringPreprocessingDataColumn, string> stringAction = null, Action<DateTimePreprocessingDataColumn, DateTime?> dateTimeAction = null) {
-      ColumnTypeSwitchAction(dataColumns[columnIndex], value, doubleAction, stringAction, dateTimeAction);
-    }
-    private void ColumnTypeSwitchAction<T>(PreprocessingDataColumn column, T value, Action<DoublePreprocessingDataColumn, double?> doubleAction,
-      Action<StringPreprocessingDataColumn, string> stringAction = null, Action<DateTimePreprocessingDataColumn, DateTime?> dateTimeAction = null) {
-      var doubleColumn = column as DoublePreprocessingDataColumn;
-      if (doubleColumn != null && doubleAction != null) doubleAction(doubleColumn, Convert<double?>(value));
-      var stringColumn = column as StringPreprocessingDataColumn;
-      if (stringColumn != null && stringAction != null) stringAction(stringColumn, Convert<string>(value));
-      var dateTimeColumn = column as DateTimePreprocessingDataColumn;
-      if (dateTimeColumn != null && dateTimeAction != null) dateTimeAction(dateTimeColumn, Convert<DateTime?>(value));
-    }
-
-    private void ColumnTypeSwitchAction(int columnIndex, Action<DoublePreprocessingDataColumn> doubleAction,
-      Action<StringPreprocessingDataColumn> stringAction = null, Action<DateTimePreprocessingDataColumn> dateTimeAction = null) {
-      ColumnTypeSwitchAction(dataColumns[columnIndex], doubleAction, stringAction, dateTimeAction);
-    }
-    private void ColumnTypeSwitchAction(PreprocessingDataColumn column, Action<DoublePreprocessingDataColumn> doubleAction,
-      Action<StringPreprocessingDataColumn> stringAction = null, Action<DateTimePreprocessingDataColumn> dateTimeAction = null) {
-      var doubleColumn = column as DoublePreprocessingDataColumn;
-      if (doubleColumn != null && doubleAction != null) doubleAction(doubleColumn);
-      var stringColumn = column as StringPreprocessingDataColumn;
-      if (stringColumn != null && stringAction != null) stringAction(stringColumn);
-      var dateTimeColumn = column as DateTimePreprocessingDataColumn;
-      if (dateTimeColumn != null && dateTimeAction != null) dateTimeAction(dateTimeColumn);
-    }
-
-
-    private T ColumnTypeSwitchFunc<T>(int columnIndex, Func<DoublePreprocessingDataColumn, double?> doubleFunc,
-      Func<StringPreprocessingDataColumn, string> stringFunc = null, Func<DateTimePreprocessingDataColumn, DateTime?> dateTimeFunc = null) {
-      var doubleColumn = dataColumns[columnIndex] as DoublePreprocessingDataColumn;
-      if (doubleColumn != null && doubleFunc != null) return Convert<T>(doubleFunc(doubleColumn));
-      var stringColumn = dataColumns[columnIndex] as StringPreprocessingDataColumn;
-      if (stringColumn != null && stringFunc != null) return Convert<T>(stringFunc(stringColumn));
-      var dateTimeColumn = dataColumns[columnIndex] as DateTimePreprocessingDataColumn;
-      if (dateTimeColumn != null && dateTimeFunc != null) return Convert<T>(dateTimeFunc(dateTimeColumn));
-      throw new InvalidOperationException("Invalid data column type.");
-    }
-
-    private T ColumnTypeSwitchFuncResult<T>(int columnIndex, Func<DoublePreprocessingDataColumn, T> doubleFunc,
-      Func<StringPreprocessingDataColumn, T> stringFunc = null, Func<DateTimePreprocessingDataColumn, T> dateTimeFunc = null) {
-      var doubleColumn = dataColumns[columnIndex] as DoublePreprocessingDataColumn;
-      if (doubleColumn != null && doubleFunc != null) return doubleFunc(doubleColumn);
-      var stringColumn = dataColumns[columnIndex] as StringPreprocessingDataColumn;
-      if (stringColumn != null && stringFunc != null) return stringFunc(stringColumn);
-      var dateTimeColumn = dataColumns[columnIndex] as DateTimePreprocessingDataColumn;
-      if (dateTimeColumn != null && dateTimeFunc != null) return dateTimeFunc(dateTimeColumn);
-      throw new InvalidOperationException("Invalid data column type.");
-    }
-    private TOut ColumnTypeSwitchFuncResult<TIn, TOut>(int columnIndex, TIn value, Func<DoublePreprocessingDataColumn, double?, TOut> doubleFunc,
-     Func<StringPreprocessingDataColumn, string, TOut> stringFunc = null, Func<DateTimePreprocessingDataColumn, DateTime?, TOut> dateTimeFunc = null) {
-      var doubleColumn = dataColumns[columnIndex] as DoublePreprocessingDataColumn;
-      if (doubleColumn != null && doubleFunc != null) return doubleFunc(doubleColumn, Convert<double?>(value));
-      var stringColumn = dataColumns[columnIndex] as StringPreprocessingDataColumn;
-      if (stringColumn != null && stringFunc != null) return stringFunc(stringColumn, Convert<string>(value));
-      var dateTimeColumn = dataColumns[columnIndex] as DateTimePreprocessingDataColumn;
-      if (dateTimeColumn != null && dateTimeFunc != null) return dateTimeFunc(dateTimeColumn, Convert<DateTime?>(value));
-      throw new InvalidOperationException("Invalid data column type.");
-    }
-
-    private IList<T> ColumnTypeSwitchFuncList<T>(int columnIndex, Func<DoublePreprocessingDataColumn, IList<double>> doubleFunc,
-      Func<StringPreprocessingDataColumn, IList<string>> stringFunc = null, Func<DateTimePreprocessingDataColumn, IList<DateTime>> dateTimeFunc = null) {
-      var doubleColumn = dataColumns[columnIndex] as DoublePreprocessingDataColumn;
-      if (doubleColumn != null && doubleFunc != null) return Convert<IList<T>>(doubleFunc(doubleColumn));
-      var stringColumn = dataColumns[columnIndex] as StringPreprocessingDataColumn;
-      if (stringColumn != null && stringFunc != null) return Convert<IList<T>>(stringFunc(stringColumn));
-      var dateTimeColumn = dataColumns[columnIndex] as DateTimePreprocessingDataColumn;
-      if (dateTimeColumn != null && dateTimeFunc != null) return Convert<IList<T>>(dateTimeFunc(dateTimeColumn));
-      throw new InvalidOperationException("Invalid data column type.");
-    }
-    private static T Convert<T>(object obj) { return (T)obj; }
-
-
     public T GetCell<T>(int columnIndex, int rowIndex) {
-      return ColumnTypeSwitchFunc<T>(columnIndex,
+      return dataColumns[columnIndex].TypeSwitch<T>(
         c => c[rowIndex],
         c => c[rowIndex],
@@ -188,5 +118,5 @@
         InsertColumn<T>(i.ToString(), i);
 
-      ColumnTypeSwitchAction<T>(columnIndex, value,
+      dataColumns[columnIndex].TypeSwitch<T>(value,
         (c, v) => c[rowIndex] = v,
         (c, v) => c[rowIndex] = v,
@@ -201,22 +131,12 @@
     }
 
-    public IList<T> GetValues<T>(int columnIndex, bool considerSelection) {
-      if (considerSelection) {
-        var list = new List<T>();
-        foreach (var rowIdx in selection[columnIndex]) {
-          list.Add(GetCell<T>(columnIndex, rowIdx));
-          //list.Add((T)dataColumns[columnIndex][rowIdx]);
-        }
-        return list;
-      } else {
-        return ColumnTypeSwitchFuncList<T>(columnIndex,
-          c => c.Values.Select(x => x ?? double.NaN).ToList(),
-          c => c.Values,
-          c => c.Values.Select(x => x ?? DateTime.MinValue).ToList());
-        //(IList<T>)dataColumns[columnIndex];
-      }
-    }
-
-    public void SetValues<T>(int columnIndex, IList<T> values) {
+    public IEnumerable<T> GetValues<T>(int columnIndex, bool considerSelection) {
+      return dataColumns[columnIndex].TypeSwitch<T>(
+        c => c.GetValues(considerSelection ? selection[columnIndex] : null),
+        c => c.GetValues(considerSelection ? selection[columnIndex] : null),
+        c => c.GetValues(considerSelection ? selection[columnIndex] : null));
+    }
+
+    public void SetValues<T>(int columnIndex, IEnumerable<T> values) {
       SaveSnapshot(DataPreprocessingChangedEventType.ChangeColumn, columnIndex, -1);
       if (VariableHasType<T>(columnIndex)) {
@@ -239,32 +159,11 @@
 
     public bool SetValue(string value, int columnIndex, int rowIndex) {
-      bool valid = false;
-      if (VariableHasType<double>(columnIndex)) {
-        double val;
-        if (string.IsNullOrWhiteSpace(value)) {
-          val = double.NaN;
-          valid = true;
-        } else {
-          valid = double.TryParse(value, out val);
-        }
-        if (valid)
-          SetCell(columnIndex, rowIndex, val);
-      } else if (VariableHasType<string>(columnIndex)) {
-        valid = value != null;
-        if (valid)
-          SetCell(columnIndex, rowIndex, value);
-      } else if (VariableHasType<DateTime>(columnIndex)) {
-        DateTime date;
-        valid = DateTime.TryParse(value, out date);
-        if (valid)
-          SetCell(columnIndex, rowIndex, date);
-      } else {
-        throw new ArgumentException("column " + columnIndex + " contains a non supported type.");
-      }
+      var column = dataColumns[columnIndex];
+      bool successful = column.SetValue(value, rowIndex);
 
       if (!IsInTransaction)
         OnChanged(DataPreprocessingChangedEventType.ChangeColumn, columnIndex, -1);
 
-      return valid;
+      return successful;
     }
 
@@ -274,5 +173,5 @@
 
     public int Rows {
-      get { return dataColumns.Count > 0 ? dataColumns[0].Length : 0; }
+      get { return dataColumns.Any() ? dataColumns.Max(c => c.Length) : 0; }
     }
     #endregion
@@ -281,12 +180,12 @@
     public void InsertRow(int rowIndex) {
       SaveSnapshot(DataPreprocessingChangedEventType.DeleteRow, -1, rowIndex);
+
       foreach (var column in dataColumns) {
-        ColumnTypeSwitchAction(column,
+        column.TypeSwitch(
+          c => c.Values.Insert(rowIndex, double.NaN),
           c => c.Values.Insert(rowIndex, null),
-          c => c.Values.Insert(rowIndex, null),
-          c => c.Values.Insert(rowIndex, null));
-        //var valueType = column.GetValueType();
-        //column.Insert(rowIndex, valueType.IsValueType ? Activator.CreateInstance(valueType) : null);
-      }
+          c => c.Values.Insert(rowIndex, DateTime.MinValue));
+      }
+
       if (TrainingPartition.Start <= rowIndex && rowIndex <= TrainingPartition.End) {
         TrainingPartition.End++;
@@ -302,42 +201,23 @@
         }
       }
+
       if (!IsInTransaction)
         OnChanged(DataPreprocessingChangedEventType.AddRow, -1, rowIndex);
     }
+
     public void DeleteRow(int rowIndex) {
-      SaveSnapshot(DataPreprocessingChangedEventType.AddRow, -1, rowIndex);
-      foreach (var column in dataColumns) {
-        ColumnTypeSwitchAction(column,
-          c => c.Values.RemoveAt(rowIndex),
-          c => c.Values.RemoveAt(rowIndex),
-          c => c.Values.RemoveAt(rowIndex));
-        //column.RemoveAt(rowIndex);
-      }
-      if (TrainingPartition.Start <= rowIndex && rowIndex <= TrainingPartition.End) {
-        TrainingPartition.End--;
-        if (TrainingPartition.End <= TestPartition.Start) {
-          TestPartition.Start--;
-          TestPartition.End--;
-        }
-      } else if (TestPartition.Start <= rowIndex && rowIndex <= TestPartition.End) {
-        TestPartition.End--;
-        if (TestPartition.End <= TrainingPartition.Start) {
-          TestPartition.Start--;
-          TestPartition.End--;
-        }
-      }
-      if (!IsInTransaction)
-        OnChanged(DataPreprocessingChangedEventType.DeleteRow, -1, rowIndex);
-    }
-    public void DeleteRowsWithIndices(IEnumerable<int> rows) {
+      DeleteRows(new[] { rowIndex });
+    }
+    public void DeleteRows(IEnumerable<int> rowIndices) {
       SaveSnapshot(DataPreprocessingChangedEventType.AddRow, -1, -1);
-      foreach (int rowIndex in rows.OrderByDescending(x => x)) {
+
+      foreach (int rowIndex in rowIndices.OrderByDescending(x => x)) {
         foreach (var column in dataColumns) {
-          ColumnTypeSwitchAction(column,
+          column.TypeSwitch(
             c => c.Values.RemoveAt(rowIndex),
             c => c.Values.RemoveAt(rowIndex),
             c => c.Values.RemoveAt(rowIndex));
-          //column.RemoveAt(rowIndex);
-        }
+        }
+
         if (TrainingPartition.Start <= rowIndex && rowIndex <= TrainingPartition.End) {
           TrainingPartition.End--;
@@ -354,4 +234,5 @@
         }
       }
+
       if (!IsInTransaction)
         OnChanged(DataPreprocessingChangedEventType.DeleteRow, -1, -1);
@@ -362,15 +243,13 @@
 
       if (typeof(T) == typeof(double)) {
-        dataColumns.Insert(columnIndex, new DoublePreprocessingDataColumn(variableName, Enumerable.Repeat<double?>(null, Rows)));
+        dataColumns.Insert(columnIndex, new DoublePreprocessingDataColumn(variableName, Enumerable.Repeat<double>(double.NaN, Rows)));
       } else if (typeof(T) == typeof(string)) {
-        dataColumns.Add(new StringPreprocessingDataColumn(variableName, Enumerable.Repeat<string>(null, Rows)));
+        dataColumns.Insert(columnIndex, new StringPreprocessingDataColumn(variableName, Enumerable.Repeat<string>(string.Empty, Rows)));
       } else if (typeof(T) == typeof(DateTime)) {
-        dataColumns.Add(new DateTimePreprocessingDataColumn(variableName, Enumerable.Repeat<DateTime?>(null, Rows)));
+        dataColumns.Insert(columnIndex, new DateTimePreprocessingDataColumn(variableName, Enumerable.Repeat<DateTime>(DateTime.MinValue, Rows)));
       } else {
         throw new ArgumentException("The datatype of column " + variableName + " must be of type double, string or DateTime");
       }
 
-      //dataColumns.Insert(columnIndex, new List<T>(Enumerable.Repeat(default(T), Rows)));
-      //variableNames.Insert(columnIndex, variableName);
       if (!IsInTransaction)
         OnChanged(DataPreprocessingChangedEventType.AddColumn, columnIndex, -1);
@@ -379,6 +258,7 @@
     public void DeleteColumn(int columnIndex) {
       SaveSnapshot(DataPreprocessingChangedEventType.AddColumn, columnIndex, -1);
+
       dataColumns.RemoveAt(columnIndex);
-      //variableNames.RemoveAt(columnIndex);
+
       if (!IsInTransaction)
         OnChanged(DataPreprocessingChangedEventType.DeleteColumn, columnIndex, -1);
@@ -386,7 +266,9 @@
 
     public void RenameColumn(int columnIndex, string name) {
-      SaveSnapshot(DataPreprocessingChangedEventType.ChangeColumn, columnIndex, -1);
       if (columnIndex < 0 || columnIndex > dataColumns.Count)
         throw new ArgumentOutOfRangeException("columnIndex");
+
+      SaveSnapshot(DataPreprocessingChangedEventType.ChangeColumn, columnIndex, -1);
+
       dataColumns[columnIndex].Name = name;
 
@@ -400,4 +282,5 @@
 
       SaveSnapshot(DataPreprocessingChangedEventType.ChangeColumn, -1, -1);
+
       for (int i = 0; i < names.Count; i++)
         dataColumns[i].Name = names[i];
@@ -408,5 +291,5 @@
 
     public bool AreAllStringColumns(IEnumerable<int> columnIndices) {
-      return columnIndices.All(x => VariableHasType<string>(x));
+      return columnIndices.All(VariableHasType<string>);
     }
     #endregion
@@ -522,7 +405,7 @@
         var stringColumn = dataColumns[i] as StringPreprocessingDataColumn;
         var dateTimeColumn = dataColumns[i] as DateTimePreprocessingDataColumn;
-        if (doubleColumn != null) values.Add(new List<double>(doubleColumn.Values.Select(x => x ?? double.NaN)));
-        else if (stringColumn != null) values.Add(new List<string>(stringColumn.Values));
-        else if (dateTimeColumn != null) values.Add(new List<DateTime>(dateTimeColumn.Values.Select(x => x ?? DateTime.MinValue)));
+        if (doubleColumn != null) values.Add(new List<double>(doubleColumn.GetValues()));
+        else if (stringColumn != null) values.Add(new List<string>(stringColumn.GetValues()));
+        else if (dateTimeColumn != null) values.Add(new List<DateTime>(dateTimeColumn.GetValues()));
         else throw new InvalidOperationException("Column type not supported for export");
       }
@@ -638,142 +521,131 @@
     #endregion
 
-    #region Statistics
-    public T GetMin<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
-      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
-      return values.Any() ? values.Min() : emptyValue;
-    }
-
-    public T GetMax<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
-      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
-      return values.Any() ? values.Max() : emptyValue;
-    }
-
-    public T GetMean<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
-      if (typeof(T) == typeof(double)) {
-        var values = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
-        return values.Any() ? Convert<T>(values.Average()) : emptyValue;
-      }
-      if (typeof(T) == typeof(string)) {
-        return Convert<T>(string.Empty);
-      }
-      if (typeof(T) == typeof(DateTime)) {
-        var values = GetValuesWithoutMissingValues<DateTime>(columnIndex, considerSelection);
-        return values.Any() ? Convert<T>(AggregateAsDouble(values, Enumerable.Average)) : emptyValue;
-      }
-
-      throw new InvalidOperationException(typeof(T) + " not supported");
-    }
-
-    public T GetMedian<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IComparable<T> {
-      if (typeof(T) == typeof(double)) {// IEnumerable<double> is faster  
-        var doubleValues = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
-        return doubleValues.Any() ? Convert<T>(doubleValues.Median()) : emptyValue;
-      }
-      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
-      return values.Any() ? values.Quantile(0.5) : emptyValue;
-    }
-
-    public T GetMode<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IEquatable<T> {
-      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
-      return values.Any() ? values.GroupBy(x => x).OrderByDescending(g => g.Count()).Select(g => g.Key).First() : emptyValue;
-    }
-
-    public T GetStandardDeviation<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
-      if (typeof(T) == typeof(double)) {
-        var values = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
-        return values.Any() ? Convert<T>(values.StandardDeviation()) : emptyValue;
-      }
-      // For DateTime, std.dev / variance would have to be TimeSpan
-      //if (typeof(T) == typeof(DateTime)) {
-      //  var values = GetValuesWithoutMissingValues<DateTime>(columnIndex, considerSelection);
-      //  return values.Any() ? Convert<T>(AggregateAsDouble(values, EnumerableStatisticExtensions.StandardDeviation)) : emptyValue;
-      //}
-      return default(T);
-    }
-
-    public T GetVariance<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
-      if (typeof(T) == typeof(double)) {
-        var values = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
-        return values.Any() ? Convert<T>(values.Variance()) : emptyValue;
-      }
-      // DateTime variance often overflows long, thus the corresponding DateTime is invalid
-      //if (typeof(T) == typeof(DateTime)) {
-      //  var values = GetValuesWithoutMissingValues<DateTime>(columnIndex, considerSelection);
-      //  return values.Any() ? Convert<T>(AggregateAsDouble(values, EnumerableStatisticExtensions.Variance)) : emptyValue;
-      //}
-      return default(T);
-    }
-
-    public T GetQuantile<T>(double alpha, int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IComparable<T> {
-      if (typeof(T) == typeof(double)) {// IEnumerable<double> is faster  
-        var doubleValues = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
-        return doubleValues.Any() ? Convert<T>(doubleValues.Quantile(alpha)) : emptyValue;
-      }
-      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
-      return values.Any() ? values.Quantile(alpha) : emptyValue;
-    }
-
-    public int GetDistinctValues<T>(int columnIndex, bool considerSelection = false) {
-      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
-      return values.GroupBy(x => x).Count();
-    }
-
-    private IEnumerable<T> GetValuesWithoutMissingValues<T>(int columnIndex, bool considerSelection) {
-      //var doubleColumn = dataColumns[columnIndex] as DoublePreprocessingDataColumn;
-      //var stringColumn = dataColumns[columnIndex] as StringPreprocessingDataColumn;
-      //var dateTimeColumn = dataColumns[columnIndex] as DateTimePreprocessingDataColumn;
-      //return GetValues<T>(columnIndex, considerSelection).Where(x =>
-      //  doubleColumn != null ? doubleColumn.IsValidValue(Convert<double>(x))
-      //  : stringColumn != null ? stringColumn.IsValidValue(Convert<string>(x))
-      //  : dateTimeColumn != null ? dateTimeColumn.IsValidValue(Convert<DateTime>(x))
-      //  : false);
-      //!IsMissingValue(x));
-
-      return GetValues<T>(columnIndex, considerSelection).Where(x =>
-        ColumnTypeSwitchFuncResult<T, bool>(columnIndex, x,
-          (c, v) => v.HasValue && c.IsValidValue(v.Value),
-          (c, v) => c.IsValidValue(v),
-          (c, v) => v.HasValue && c.IsValidValue(v.Value)
-      ));
-    }
-
-    private static DateTime AggregateAsDouble(IEnumerable<DateTime> values, Func<IEnumerable<double>, double> func) {
-      return new DateTime((long)(func(values.Select(x => (double)x.Ticks / TimeSpan.TicksPerSecond)) * TimeSpan.TicksPerSecond));
-    }
-
-    public int GetMissingValueCount() {
-      int count = 0;
-      for (int i = 0; i < Columns; ++i) {
-        count += GetMissingValueCount(i);
-      }
-      return count;
-    }
-    public int GetMissingValueCount(int columnIndex) {
-      int sum = 0;
-      for (int i = 0; i < Rows; i++) {
-        if (IsCellEmpty(columnIndex, i))
-          sum++;
-      }
-      return sum;
-    }
-    public int GetRowMissingValueCount(int rowIndex) {
-      int sum = 0;
-      for (int i = 0; i < Columns; i++) {
-        if (IsCellEmpty(i, rowIndex))
-          sum++;
-      }
-      return sum;
-    }
-    #endregion
-
-    #region Helpers
-    private static IList<IList> CopyVariableValues(IList<IList> original) {
-      var copy = new List<IList>(original);
-      for (int i = 0; i < original.Count; ++i) {
-        copy[i] = (IList)Activator.CreateInstance(original[i].GetType(), original[i]);
-      }
-      return copy;
-    }
-    #endregion
+    /* #region Statistics
+     public T GetMin<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+       try {
+         return dataColumns[columnIndex].TypeSwitch<T>(
+           col => col.GetMin(considerSelection ? Selection[columnIndex] : null),
+           col => col.GetMin(considerSelection ? Selection[columnIndex] : null),
+           col => col.GetMin(considerSelection ? Selection[columnIndex] : null));
+       } catch (InvalidOperationException) {
+         return emptyValue;
+       }
+     }
+
+     public T GetMax<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+       var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
+       return values.Any() ? values.Max() : emptyValue;
+     }
+
+     public T GetMean<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+       return 
+
+
+       if (typeof(T) == typeof(double)) {
+         var values = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
+         return values.Any() ? Convert<T>(values.Average()) : emptyValue;
+       }
+       if (typeof(T) == typeof(string)) {
+         return Convert<T>(string.Empty);
+       }
+       if (typeof(T) == typeof(DateTime)) {
+         var values = GetValuesWithoutMissingValues<DateTime>(columnIndex, considerSelection);
+         return values.Any() ? Convert<T>(AggregateAsDouble(values, Enumerable.Average)) : emptyValue;
+       }
+
+       throw new InvalidOperationException(typeof(T) + " not supported");
+     }
+
+     public T GetMedian<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IComparable<T> {
+       if (typeof(T) == typeof(double)) {// IEnumerable<double> is faster  
+         var doubleValues = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
+         return doubleValues.Any() ? Convert<T>(doubleValues.Median()) : emptyValue;
+       }
+       var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
+       return values.Any() ? values.Quantile(0.5) : emptyValue;
+     }
+
+     public T GetMode<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IEquatable<T> {
+       var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
+       return values.Any() ? values.GroupBy(x => x).OrderByDescending(g => g.Count()).Select(g => g.Key).First() : emptyValue;
+     }
+
+     public T GetStandardDeviation<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+       if (typeof(T) == typeof(double)) {
+         var values = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
+         return values.Any() ? Convert<T>(values.StandardDeviation()) : emptyValue;
+       }
+       // For DateTime, std.dev / variance would have to be TimeSpan
+       //if (typeof(T) == typeof(DateTime)) {
+       //  var values = GetValuesWithoutMissingValues<DateTime>(columnIndex, considerSelection);
+       //  return values.Any() ? Convert<T>(AggregateAsDouble(values, EnumerableStatisticExtensions.StandardDeviation)) : emptyValue;
+       //}
+       return default(T);
+     }
+
+     public T GetVariance<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+       if (typeof(T) == typeof(double)) {
+         var values = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
+         return values.Any() ? Convert<T>(values.Variance()) : emptyValue;
+       }
+       // DateTime variance often overflows long, thus the corresponding DateTime is invalid
+       //if (typeof(T) == typeof(DateTime)) {
+       //  var values = GetValuesWithoutMissingValues<DateTime>(columnIndex, considerSelection);
+       //  return values.Any() ? Convert<T>(AggregateAsDouble(values, EnumerableStatisticExtensions.Variance)) : emptyValue;
+       //}
+       return default(T);
+     }
+
+     public T GetQuantile<T>(double alpha, int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IComparable<T> {
+       if (typeof(T) == typeof(double)) {// IEnumerable<double> is faster  
+         var doubleValues = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
+         return doubleValues.Any() ? Convert<T>(doubleValues.Quantile(alpha)) : emptyValue;
+       }
+       var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
+       return values.Any() ? values.Quantile(alpha) : emptyValue;
+     }
+
+     public int GetDistinctValues<T>(int columnIndex, bool considerSelection = false) {
+       var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
+       return values.GroupBy(x => x).Count();
+     }
+
+     private IEnumerable<T> GetValuesWithoutMissingValues<T>(int columnIndex, bool considerSelection) {
+       return GetValues<T>(columnIndex, considerSelection).Where(x =>
+         ColumnTypeSwitch<T, bool>(dataColumns[columnIndex], x,
+           (c, v) => c.IsValidValue(v),
+           (c, v) => c.IsValidValue(v),
+           (c, v) => c.IsValidValue(v)
+       ));
+     }
+
+     private static DateTime AggregateAsDouble(IEnumerable<DateTime> values, Func<IEnumerable<double>, double> func) {
+       return new DateTime((long)(func(values.Select(x => (double)x.Ticks / TimeSpan.TicksPerSecond)) * TimeSpan.TicksPerSecond));
+     }
+
+     public int GetMissingValueCount() {
+       int count = 0;
+       for (int i = 0; i < Columns; ++i) {
+         count += GetMissingValueCount(i);
+       }
+       return count;
+     }
+     public int GetMissingValueCount(int columnIndex) {
+       int sum = 0;
+       for (int i = 0; i < Rows; i++) {
+         if (IsCellEmpty(columnIndex, i))
+           sum++;
+       }
+       return sum;
+     }
+     public int GetRowMissingValueCount(int rowIndex) {
+       int sum = 0;
+       for (int i = 0; i < Columns; i++) {
+         if (IsCellEmpty(i, rowIndex))
+           sum++;
+       }
+       return sum;
+     }
+     #endregion  */
   }
 
Index: branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/PreprocessingTransformator.cs
===================================================================
--- branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/PreprocessingTransformator.cs	(revision 15291)
+++ branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/PreprocessingTransformator.cs	(revision 15309)
@@ -83,5 +83,5 @@
           int colIndex = preprocessingData.GetColumnIndex(transformation.Column);
           var originalData = preprocessingData.GetValues<double>(colIndex);
-          originalColumns.Add(transformation.Column, originalData);
+          originalColumns.Add(transformation.Column, originalData.ToList());
         }
       }
@@ -107,5 +107,5 @@
     }
 
-    private IEnumerable<double> ApplyDoubleTransformation(Transformation<double> transformation, IList<double> data, out bool success, out string errorMsg) {
+    private IEnumerable<double> ApplyDoubleTransformation(Transformation<double> transformation, IEnumerable<double> data, out bool success, out string errorMsg) {
       success = transformation.Check(data, out errorMsg);
       // don't apply when the check fails
Index: branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/ProblemDataCreator.cs
===================================================================
--- branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/ProblemDataCreator.cs	(revision 15291)
+++ branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/ProblemDataCreator.cs	(revision 15309)
@@ -129,5 +129,5 @@
     }
 
-    private bool IsNotConstantInputVariable(IList<double> list) {
+    private bool IsNotConstantInputVariable(IEnumerable<double> list) {
       return context.Data.TrainingPartition.End - context.Data.TrainingPartition.Start > 1 || list.Range() > 0;
     }
