Changeset 15291

Timestamp:

07/27/17 12:45:23 (7 years ago)

Author:

pfleck

Message:

#2809: Added (Double/String/DateTime)PreprocessingDataColumn. (experimental state)

Location:

branches/DataPreprocessing Cleanup

Files:

• DataPreprocessing Cleanup.sln (modified) (1 diff)
• HeuristicLab.DataPreprocessing/3.4/Data/Columns (added)
• HeuristicLab.DataPreprocessing/3.4/Data/Columns/DateTimePreprocessingDataColumn.cs (added)
• HeuristicLab.DataPreprocessing/3.4/Data/Columns/DoublePreprocessingDataColumn.cs (added)
• HeuristicLab.DataPreprocessing/3.4/Data/Columns/PreprocessingDataColumn.cs (added)
• HeuristicLab.DataPreprocessing/3.4/Data/Columns/StringPreprocessingDataColumn.cs (added)
• HeuristicLab.DataPreprocessing/3.4/Data/PreprocessingData.cs (modified) (27 diffs)
• HeuristicLab.DataPreprocessing/3.4/HeuristicLab.DataPreprocessing-3.4.csproj (modified) (1 diff)

branches/DataPreprocessing Cleanup/DataPreprocessing Cleanup.sln

@@ -2 +2 @@
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 15
-VisualStudioVersion = 15.0.26430.15
+VisualStudioVersion = 15.0.26430.16
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.DataPreprocessing-3.4", "HeuristicLab.DataPreprocessing\3.4\HeuristicLab.DataPreprocessing-3.4.csproj", "{3B90F866-70F8-43EF-A541-51819D255B7B}"

branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/PreprocessingData.cs

@@ -32 +32 @@
 
 namespace HeuristicLab.DataPreprocessing {
-
   [Item("PreprocessingData", "Represents data used for preprocessing.")]
   [StorableClass]
@@ -38 +37 @@
 
     [Storable]
-    protected IList<IList> variableValues;
-    [Storable]
-    protected IList<string> variableNames;
+    protected List<PreprocessingDataColumn> dataColumns;
 
     #region Constructor, Cloning & Persistence
@@ -47 +44 @@
       Name = "Preprocessing Data";
 
+      dataColumns = new List<PreprocessingDataColumn>();
       Transformations = new List<ITransformation>();
       selection = new Dictionary<int, IList<int>>();
@@ -57 +55 @@
     protected PreprocessingData(PreprocessingData original, Cloner cloner)
       : base(original, cloner) {
-      variableValues = CopyVariableValues(original.variableValues);
-      variableNames = new List<string>(original.variableNames);
-      TrainingPartition = (IntRange)original.TrainingPartition.Clone(cloner);
-      TestPartition = (IntRange)original.TestPartition.Clone(cloner);
+      dataColumns = new List<PreprocessingDataColumn>(original.dataColumns.Select(cloner.Clone));
+      TrainingPartition = cloner.Clone(original.TrainingPartition);
+      TestPartition = cloner.Clone(original.TestPartition);
       Transformations = new List<ITransformation>(original.Transformations.Select(cloner.Clone));
 
@@ -99 +96 @@
     #region Cells
     public bool IsCellEmpty(int columnIndex, int rowIndex) {
-      var value = variableValues[columnIndex][rowIndex];
-      return IsMissingValue(value);
-    }
+      return !dataColumns[columnIndex].IsValidValue(rowIndex);
+    }
+
+    private void ColumnTypeSwitchAction<T>(int columnIndex, T value, Action<DoublePreprocessingDataColumn, double?> doubleAction,
+      Action<StringPreprocessingDataColumn, string> stringAction = null, Action<DateTimePreprocessingDataColumn, DateTime?> dateTimeAction = null) {
+      ColumnTypeSwitchAction(dataColumns[columnIndex], value, doubleAction, stringAction, dateTimeAction);
+    }
+    private void ColumnTypeSwitchAction<T>(PreprocessingDataColumn column, T value, Action<DoublePreprocessingDataColumn, double?> doubleAction,
+      Action<StringPreprocessingDataColumn, string> stringAction = null, Action<DateTimePreprocessingDataColumn, DateTime?> dateTimeAction = null) {
+      var doubleColumn = column as DoublePreprocessingDataColumn;
+      if (doubleColumn != null && doubleAction != null) doubleAction(doubleColumn, Convert<double?>(value));
+      var stringColumn = column as StringPreprocessingDataColumn;
+      if (stringColumn != null && stringAction != null) stringAction(stringColumn, Convert<string>(value));
+      var dateTimeColumn = column as DateTimePreprocessingDataColumn;
+      if (dateTimeColumn != null && dateTimeAction != null) dateTimeAction(dateTimeColumn, Convert<DateTime?>(value));
+    }
+
+    private void ColumnTypeSwitchAction(int columnIndex, Action<DoublePreprocessingDataColumn> doubleAction,
+      Action<StringPreprocessingDataColumn> stringAction = null, Action<DateTimePreprocessingDataColumn> dateTimeAction = null) {
+      ColumnTypeSwitchAction(dataColumns[columnIndex], doubleAction, stringAction, dateTimeAction);
+    }
+    private void ColumnTypeSwitchAction(PreprocessingDataColumn column, Action<DoublePreprocessingDataColumn> doubleAction,
+      Action<StringPreprocessingDataColumn> stringAction = null, Action<DateTimePreprocessingDataColumn> dateTimeAction = null) {
+      var doubleColumn = column as DoublePreprocessingDataColumn;
+      if (doubleColumn != null && doubleAction != null) doubleAction(doubleColumn);
+      var stringColumn = column as StringPreprocessingDataColumn;
+      if (stringColumn != null && stringAction != null) stringAction(stringColumn);
+      var dateTimeColumn = column as DateTimePreprocessingDataColumn;
+      if (dateTimeColumn != null && dateTimeAction != null) dateTimeAction(dateTimeColumn);
+    }
+
+
+    private T ColumnTypeSwitchFunc<T>(int columnIndex, Func<DoublePreprocessingDataColumn, double?> doubleFunc,
+      Func<StringPreprocessingDataColumn, string> stringFunc = null, Func<DateTimePreprocessingDataColumn, DateTime?> dateTimeFunc = null) {
+      var doubleColumn = dataColumns[columnIndex] as DoublePreprocessingDataColumn;
+      if (doubleColumn != null && doubleFunc != null) return Convert<T>(doubleFunc(doubleColumn));
+      var stringColumn = dataColumns[columnIndex] as StringPreprocessingDataColumn;
+      if (stringColumn != null && stringFunc != null) return Convert<T>(stringFunc(stringColumn));
+      var dateTimeColumn = dataColumns[columnIndex] as DateTimePreprocessingDataColumn;
+      if (dateTimeColumn != null && dateTimeFunc != null) return Convert<T>(dateTimeFunc(dateTimeColumn));
+      throw new InvalidOperationException("Invalid data column type.");
+    }
+
+    private T ColumnTypeSwitchFuncResult<T>(int columnIndex, Func<DoublePreprocessingDataColumn, T> doubleFunc,
+      Func<StringPreprocessingDataColumn, T> stringFunc = null, Func<DateTimePreprocessingDataColumn, T> dateTimeFunc = null) {
+      var doubleColumn = dataColumns[columnIndex] as DoublePreprocessingDataColumn;
+      if (doubleColumn != null && doubleFunc != null) return doubleFunc(doubleColumn);
+      var stringColumn = dataColumns[columnIndex] as StringPreprocessingDataColumn;
+      if (stringColumn != null && stringFunc != null) return stringFunc(stringColumn);
+      var dateTimeColumn = dataColumns[columnIndex] as DateTimePreprocessingDataColumn;
+      if (dateTimeColumn != null && dateTimeFunc != null) return dateTimeFunc(dateTimeColumn);
+      throw new InvalidOperationException("Invalid data column type.");
+    }
+    private TOut ColumnTypeSwitchFuncResult<TIn, TOut>(int columnIndex, TIn value, Func<DoublePreprocessingDataColumn, double?, TOut> doubleFunc,
+     Func<StringPreprocessingDataColumn, string, TOut> stringFunc = null, Func<DateTimePreprocessingDataColumn, DateTime?, TOut> dateTimeFunc = null) {
+      var doubleColumn = dataColumns[columnIndex] as DoublePreprocessingDataColumn;
+      if (doubleColumn != null && doubleFunc != null) return doubleFunc(doubleColumn, Convert<double?>(value));
+      var stringColumn = dataColumns[columnIndex] as StringPreprocessingDataColumn;
+      if (stringColumn != null && stringFunc != null) return stringFunc(stringColumn, Convert<string>(value));
+      var dateTimeColumn = dataColumns[columnIndex] as DateTimePreprocessingDataColumn;
+      if (dateTimeColumn != null && dateTimeFunc != null) return dateTimeFunc(dateTimeColumn, Convert<DateTime?>(value));
+      throw new InvalidOperationException("Invalid data column type.");
+    }
+
+    private IList<T> ColumnTypeSwitchFuncList<T>(int columnIndex, Func<DoublePreprocessingDataColumn, IList<double>> doubleFunc,
+      Func<StringPreprocessingDataColumn, IList<string>> stringFunc = null, Func<DateTimePreprocessingDataColumn, IList<DateTime>> dateTimeFunc = null) {
+      var doubleColumn = dataColumns[columnIndex] as DoublePreprocessingDataColumn;
+      if (doubleColumn != null && doubleFunc != null) return Convert<IList<T>>(doubleFunc(doubleColumn));
+      var stringColumn = dataColumns[columnIndex] as StringPreprocessingDataColumn;
+      if (stringColumn != null && stringFunc != null) return Convert<IList<T>>(stringFunc(stringColumn));
+      var dateTimeColumn = dataColumns[columnIndex] as DateTimePreprocessingDataColumn;
+      if (dateTimeColumn != null && dateTimeFunc != null) return Convert<IList<T>>(dateTimeFunc(dateTimeColumn));
+      throw new InvalidOperationException("Invalid data column type.");
+    }
+    private static T Convert<T>(object obj) { return (T)obj; }
+
 
     public T GetCell<T>(int columnIndex, int rowIndex) {
-      return (T)variableValues[columnIndex][rowIndex];
+      return ColumnTypeSwitchFunc<T>(columnIndex,
+        c => c[rowIndex],
+        c => c[rowIndex],
+        c => c[rowIndex]);
     }
 
@@ -115 +188 @@
         InsertColumn<T>(i.ToString(), i);
 
-      variableValues[columnIndex][rowIndex] = value;
+      ColumnTypeSwitchAction<T>(columnIndex, value,
+        (c, v) => c[rowIndex] = v,
+        (c, v) => c[rowIndex] = v,
+        (c, v) => c[rowIndex] = v);
+
       if (!IsInTransaction)
         OnChanged(DataPreprocessingChangedEventType.ChangeItem, columnIndex, rowIndex);
@@ -121 +198 @@
 
     public string GetCellAsString(int columnIndex, int rowIndex) {
-      return variableValues[columnIndex][rowIndex].ToString();
+      return dataColumns[columnIndex].GetValue(rowIndex);
     }
 
@@ -128 +205 @@
         var list = new List<T>();
         foreach (var rowIdx in selection[columnIndex]) {
-          list.Add((T)variableValues[columnIndex][rowIdx]);
+          list.Add(GetCell<T>(columnIndex, rowIdx));
+          //list.Add((T)dataColumns[columnIndex][rowIdx]);
         }
         return list;
       } else {
-        return (IList<T>)variableValues[columnIndex];
+        return ColumnTypeSwitchFuncList<T>(columnIndex,
+          c => c.Values.Select(x => x ?? double.NaN).ToList(),
+          c => c.Values,
+          c => c.Values.Select(x => x ?? DateTime.MinValue).ToList());
+        //(IList<T>)dataColumns[columnIndex];
       }
     }
@@ -139 +221 @@
       SaveSnapshot(DataPreprocessingChangedEventType.ChangeColumn, columnIndex, -1);
       if (VariableHasType<T>(columnIndex)) {
-        variableValues[columnIndex] = (IList)values;
+        var name = dataColumns[columnIndex].Name;
+        if (dataColumns[columnIndex].IsType<double>()) {
+          dataColumns[columnIndex] = new DoublePreprocessingDataColumn(name, (IEnumerable<double>)values);
+        } else if (dataColumns[columnIndex].IsType<string>()) {
+          dataColumns[columnIndex] = new StringPreprocessingDataColumn(name, (IEnumerable<string>)values);
+        } else if (dataColumns[columnIndex].IsType<DateTime>()) {
+          dataColumns[columnIndex] = new DateTimePreprocessingDataColumn(name, (IEnumerable<DateTime>)values);
+        } else {
+          throw new ArgumentException("Unknown column type");
+        }
       } else {
-        throw new ArgumentException("The datatype of column " + columnIndex + " must be of type " + variableValues[columnIndex].GetType().Name + " but was " + typeof(T).Name);
+        throw new ArgumentException("The datatype of column " + columnIndex + " must be of type " + dataColumns[columnIndex].GetType().Name + " but was " + typeof(T).Name);
       }
       if (!IsInTransaction)
@@ -179 +270 @@
 
     public int Columns {
-      get { return variableNames.Count; }
+      get { return dataColumns.Count; }
     }
 
     public int Rows {
-      get { return variableValues.Count > 0 ? variableValues[0].Count : 0; }
-    }
-
-    public static bool IsMissingValue(object value) {
-      if (value is double) return double.IsNaN((double)value);
-      if (value is string) return string.IsNullOrEmpty((string)value);
-      if (value is DateTime) return ((DateTime)value).Equals(DateTime.MinValue);
-      throw new ArgumentException();
+      get { return dataColumns.Count > 0 ? dataColumns[0].Length : 0; }
     }
     #endregion
@@ -197 +281 @@
     public void InsertRow(int rowIndex) {
       SaveSnapshot(DataPreprocessingChangedEventType.DeleteRow, -1, rowIndex);
-      foreach (IList column in variableValues) {
-        Type type = column.GetType().GetGenericArguments()[0];
-        column.Insert(rowIndex, type.IsValueType ? Activator.CreateInstance(type) : null);
+      foreach (var column in dataColumns) {
+        ColumnTypeSwitchAction(column,
+          c => c.Values.Insert(rowIndex, null),
+          c => c.Values.Insert(rowIndex, null),
+          c => c.Values.Insert(rowIndex, null));
+        //var valueType = column.GetValueType();
+        //column.Insert(rowIndex, valueType.IsValueType ? Activator.CreateInstance(valueType) : null);
       }
       if (TrainingPartition.Start <= rowIndex && rowIndex <= TrainingPartition.End) {
@@ -219 +307 @@
     public void DeleteRow(int rowIndex) {
       SaveSnapshot(DataPreprocessingChangedEventType.AddRow, -1, rowIndex);
-      foreach (IList column in variableValues) {
-        column.RemoveAt(rowIndex);
+      foreach (var column in dataColumns) {
+        ColumnTypeSwitchAction(column,
+          c => c.Values.RemoveAt(rowIndex),
+          c => c.Values.RemoveAt(rowIndex),
+          c => c.Values.RemoveAt(rowIndex));
+        //column.RemoveAt(rowIndex);
       }
       if (TrainingPartition.Start <= rowIndex && rowIndex <= TrainingPartition.End) {
@@ -241 +333 @@
       SaveSnapshot(DataPreprocessingChangedEventType.AddRow, -1, -1);
       foreach (int rowIndex in rows.OrderByDescending(x => x)) {
-        foreach (IList column in variableValues) {
-          column.RemoveAt(rowIndex);
+        foreach (var column in dataColumns) {
+          ColumnTypeSwitchAction(column,
+            c => c.Values.RemoveAt(rowIndex),
+            c => c.Values.RemoveAt(rowIndex),
+            c => c.Values.RemoveAt(rowIndex));
+          //column.RemoveAt(rowIndex);
         }
         if (TrainingPartition.Start <= rowIndex && rowIndex <= TrainingPartition.End) {
@@ -264 +360 @@
     public void InsertColumn<T>(string variableName, int columnIndex) {
       SaveSnapshot(DataPreprocessingChangedEventType.DeleteColumn, columnIndex, -1);
-      variableValues.Insert(columnIndex, new List<T>(Enumerable.Repeat(default(T), Rows)));
-      variableNames.Insert(columnIndex, variableName);
+
+      if (typeof(T) == typeof(double)) {
+        dataColumns.Insert(columnIndex, new DoublePreprocessingDataColumn(variableName, Enumerable.Repeat<double?>(null, Rows)));
+      } else if (typeof(T) == typeof(string)) {
+        dataColumns.Add(new StringPreprocessingDataColumn(variableName, Enumerable.Repeat<string>(null, Rows)));
+      } else if (typeof(T) == typeof(DateTime)) {
+        dataColumns.Add(new DateTimePreprocessingDataColumn(variableName, Enumerable.Repeat<DateTime?>(null, Rows)));
+      } else {
+        throw new ArgumentException("The datatype of column " + variableName + " must be of type double, string or DateTime");
+      }
+
+      //dataColumns.Insert(columnIndex, new List<T>(Enumerable.Repeat(default(T), Rows)));
+      //variableNames.Insert(columnIndex, variableName);
       if (!IsInTransaction)
         OnChanged(DataPreprocessingChangedEventType.AddColumn, columnIndex, -1);
@@ -272 +379 @@
     public void DeleteColumn(int columnIndex) {
       SaveSnapshot(DataPreprocessingChangedEventType.AddColumn, columnIndex, -1);
-      variableValues.RemoveAt(columnIndex);
-      variableNames.RemoveAt(columnIndex);
+      dataColumns.RemoveAt(columnIndex);
+      //variableNames.RemoveAt(columnIndex);
       if (!IsInTransaction)
         OnChanged(DataPreprocessingChangedEventType.DeleteColumn, columnIndex, -1);
@@ -280 +387 @@
     public void RenameColumn(int columnIndex, string name) {
       SaveSnapshot(DataPreprocessingChangedEventType.ChangeColumn, columnIndex, -1);
-      if (columnIndex < 0 || columnIndex > variableNames.Count)
+      if (columnIndex < 0 || columnIndex > dataColumns.Count)
         throw new ArgumentOutOfRangeException("columnIndex");
-      variableNames[columnIndex] = name;
+      dataColumns[columnIndex].Name = name;
 
       if (!IsInTransaction)
@@ -290 +397 @@
     public void RenameColumns(IList<string> names) {
       if (names == null) throw new ArgumentNullException("names");
-      if (names.Count != variableNames.Count) throw new ArgumentException("number of names must match the number of columns.", "names");
+      if (names.Count != dataColumns.Count) throw new ArgumentException("number of names must match the number of columns.", "names");
 
       SaveSnapshot(DataPreprocessingChangedEventType.ChangeColumn, -1, -1);
       for (int i = 0; i < names.Count; i++)
-        variableNames[i] = names[i];
+        dataColumns[i].Name = names[i];
 
       if (!IsInTransaction)
@@ -307 +414 @@
     #region Variables
     public IEnumerable<string> VariableNames {
-      get { return variableNames; }
+      get { return dataColumns.Select(c => c.Name); }
     }
 
     public IEnumerable<string> GetDoubleVariableNames() {
-      var doubleVariableNames = new List<string>();
-      for (int i = 0; i < Columns; ++i) {
-        if (VariableHasType<double>(i)) {
-          doubleVariableNames.Add(variableNames[i]);
-        }
-      }
-      return doubleVariableNames;
+      return dataColumns.OfType<DoublePreprocessingDataColumn>().Select(c => c.Name);
     }
 
     public string GetVariableName(int columnIndex) {
-      return variableNames[columnIndex];
+      return dataColumns[columnIndex].Name;
     }
 
     public int GetColumnIndex(string variableName) {
-      return variableNames.IndexOf(variableName);
+      return dataColumns.FindIndex(c => c.Name == variableName);
     }
 
     public bool VariableHasType<T>(int columnIndex) {
-      return columnIndex >= variableValues.Count || variableValues[columnIndex] is List<T>;
+      return dataColumns[columnIndex].IsType<T>();
     }
 
     public Type GetVariableType(int columnIndex) {
-      var listType = variableValues[columnIndex].GetType();
-      return listType.GenericTypeArguments.Single();
+      return dataColumns[columnIndex].GetValueType();
     }
 
@@ -392 +492 @@
     #region Import & Export
     public void Import(IDataAnalysisProblemData problemData) {
-      Dataset dataset = (Dataset)problemData.Dataset;
-      variableNames = new List<string>(problemData.Dataset.VariableNames);
+      var dataset = problemData.Dataset;
       InputVariables = new List<string>(problemData.AllowedInputVariables);
-      TargetVariable = (problemData is IRegressionProblemData) ? ((IRegressionProblemData)problemData).TargetVariable
-        : (problemData is IClassificationProblemData) ? ((IClassificationProblemData)problemData).TargetVariable
-          : null;
-
-      int columnIndex = 0;
-      variableValues = new List<IList>();
+      TargetVariable = problemData is IRegressionProblemData ? ((IRegressionProblemData)problemData).TargetVariable
+        : problemData is IClassificationProblemData ? ((IClassificationProblemData)problemData).TargetVariable
+        : null;
+
+      dataColumns.Clear();
       foreach (var variableName in problemData.Dataset.VariableNames) {
         if (dataset.VariableHasType<double>(variableName)) {
-          variableValues.Insert(columnIndex, dataset.GetDoubleValues(variableName).ToList());
+          dataColumns.Add(new DoublePreprocessingDataColumn(variableName, dataset.GetDoubleValues(variableName)));
         } else if (dataset.VariableHasType<string>(variableName)) {
-          variableValues.Insert(columnIndex, dataset.GetStringValues(variableName).ToList());
+          dataColumns.Add(new StringPreprocessingDataColumn(variableName, dataset.GetStringValues(variableName)));
         } else if (dataset.VariableHasType<DateTime>(variableName)) {
-          variableValues.Insert(columnIndex, dataset.GetDateTimeValues(variableName).ToList());
+          dataColumns.Add(new DateTimePreprocessingDataColumn(variableName, dataset.GetDateTimeValues(variableName)));
         } else {
           throw new ArgumentException("The datatype of column " + variableName + " must be of type double, string or DateTime");
         }
-        ++columnIndex;
       }
 
@@ -421 +518 @@
       IList<IList> values = new List<IList>();
 
-      for (int i = 0; i < Columns; ++i) {
-        values.Add(variableValues[i]);
-      }
-
-      var dataset = new Dataset(variableNames, values);
-      return dataset;
+      for (int i = 0; i < Columns; i++) {
+        var doubleColumn = dataColumns[i] as DoublePreprocessingDataColumn;
+        var stringColumn = dataColumns[i] as StringPreprocessingDataColumn;
+        var dateTimeColumn = dataColumns[i] as DateTimePreprocessingDataColumn;
+        if (doubleColumn != null) values.Add(new List<double>(doubleColumn.Values.Select(x => x ?? double.NaN)));
+        else if (stringColumn != null) values.Add(new List<string>(stringColumn.Values));
+        else if (dateTimeColumn != null) values.Add(new List<DateTime>(dateTimeColumn.Values.Select(x => x ?? DateTime.MinValue)));
+        else throw new InvalidOperationException("Column type not supported for export");
+      }
+
+      return new Dataset(VariableNames, values);
     }
     #endregion
@@ -452 +554 @@
 
     #region Transactions
-    // Stapshot/History are nost storable/cloneable on purpose
+    // Snapshot/History are not storable/cloneable on purpose
     private class Snapshot {
-      public IList<IList> VariableValues { get; set; }
-      public IList<string> VariableNames { get; set; }
+      public List<PreprocessingDataColumn> DataColumns { get; set; }
 
       public IntRange TrainingPartition { get; set; }
@@ -472 +573 @@
     }
 
-    private const int MAX_UNDO_DEPTH = 5;
+    private const int MaxUndoDepth = 5;
 
     private readonly IList<Snapshot> undoHistory = new List<Snapshot>();
@@ -482 +583 @@
       if (IsInTransaction) return;
 
+      var cloner = new Cloner();
       var currentSnapshot = new Snapshot {
-        VariableValues = CopyVariableValues(variableValues),
-        VariableNames = new List<string>(variableNames),
+        DataColumns = new List<PreprocessingDataColumn>(dataColumns.Select(cloner.Clone)),
         TrainingPartition = new IntRange(TrainingPartition.Start, TrainingPartition.End),
         TestPartition = new IntRange(TestPartition.Start, TestPartition.End),
@@ -493 +594 @@
       };
 
-      if (undoHistory.Count >= MAX_UNDO_DEPTH)
+      if (undoHistory.Count >= MaxUndoDepth)
         undoHistory.RemoveAt(0);
 
@@ -506 +607 @@
       if (IsUndoAvailable) {
         Snapshot previousSnapshot = undoHistory[undoHistory.Count - 1];
-        variableValues = previousSnapshot.VariableValues;
-        variableNames = previousSnapshot.VariableNames;
+        dataColumns = previousSnapshot.DataColumns;
         TrainingPartition = previousSnapshot.TrainingPartition;
         TestPartition = previousSnapshot.TestPartition;
@@ -620 +720 @@
 
     private IEnumerable<T> GetValuesWithoutMissingValues<T>(int columnIndex, bool considerSelection) {
-      return GetValues<T>(columnIndex, considerSelection).Where(x => !IsMissingValue(x));
+      //var doubleColumn = dataColumns[columnIndex] as DoublePreprocessingDataColumn;
+      //var stringColumn = dataColumns[columnIndex] as StringPreprocessingDataColumn;
+      //var dateTimeColumn = dataColumns[columnIndex] as DateTimePreprocessingDataColumn;
+      //return GetValues<T>(columnIndex, considerSelection).Where(x =>
+      //  doubleColumn != null ? doubleColumn.IsValidValue(Convert<double>(x))
+      //  : stringColumn != null ? stringColumn.IsValidValue(Convert<string>(x))
+      //  : dateTimeColumn != null ? dateTimeColumn.IsValidValue(Convert<DateTime>(x))
+      //  : false);
+      //!IsMissingValue(x));
+
+      return GetValues<T>(columnIndex, considerSelection).Where(x =>
+        ColumnTypeSwitchFuncResult<T, bool>(columnIndex, x,
+          (c, v) => v.HasValue && c.IsValidValue(v.Value),
+          (c, v) => c.IsValidValue(v),
+          (c, v) => v.HasValue && c.IsValidValue(v.Value)
+      ));
     }
 
@@ -626 +741 @@
       return new DateTime((long)(func(values.Select(x => (double)x.Ticks / TimeSpan.TicksPerSecond)) * TimeSpan.TicksPerSecond));
     }
-    private static T Convert<T>(object obj) { return (T)obj; }
 
     public int GetMissingValueCount() {

branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/HeuristicLab.DataPreprocessing-3.4.csproj

@@ -123 +123 @@
     <Compile Include="Content\ScatterPlotContent.cs" />
     <Compile Include="Content\DataCompletenessChartContent.cs" />
+    <Compile Include="Data\Columns\DateTimePreprocessingDataColumn.cs" />
+    <Compile Include="Data\Columns\DoublePreprocessingDataColumn.cs" />
+    <Compile Include="Data\Columns\PreprocessingDataColumn.cs" />
+    <Compile Include="Data\Columns\StringPreprocessingDataColumn.cs" />
     <Compile Include="Data\FilteredPreprocessingData.cs" />
     <Compile Include="Content\ManipulationContent.cs" />