source: branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/PreprocessingData.cs @ 15291

#region License Information
/* HeuristicLab
 * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */
#endregion

using System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
using HeuristicLab.Problems.DataAnalysis;

namespace HeuristicLab.DataPreprocessing {
  [Item("PreprocessingData", "Represents data used for preprocessing.")]
  [StorableClass]
  public class PreprocessingData : NamedItem, IPreprocessingData {

    [Storable]
    protected List<PreprocessingDataColumn> dataColumns;

    #region Constructor, Cloning & Persistence
    public PreprocessingData(IDataAnalysisProblemData problemData)
      : base() {
      Name = "Preprocessing Data";

      dataColumns = new List<PreprocessingDataColumn>();
      Transformations = new List<ITransformation>();
      selection = new Dictionary<int, IList<int>>();

      Import(problemData);

      RegisterEventHandler();
    }

    protected PreprocessingData(PreprocessingData original, Cloner cloner)
      : base(original, cloner) {
      dataColumns = new List<PreprocessingDataColumn>(original.dataColumns.Select(cloner.Clone));
      TrainingPartition = cloner.Clone(original.TrainingPartition);
      TestPartition = cloner.Clone(original.TestPartition);
      Transformations = new List<ITransformation>(original.Transformations.Select(cloner.Clone));

      InputVariables = new List<string>(original.InputVariables);
      TargetVariable = original.TargetVariable;

      RegisterEventHandler();
    }
    public override IDeepCloneable Clone(Cloner cloner) {
      return new PreprocessingData(this, cloner);
    }

    [StorableConstructor]
    protected PreprocessingData(bool deserializing)
      : base(deserializing) { }
    [StorableHook(HookType.AfterDeserialization)]
    private void AfterDeserialization() {
      RegisterEventHandler();
    }

    private void RegisterEventHandler() {
      Changed += (s, e) => {
        switch (e.Type) {
          case DataPreprocessingChangedEventType.DeleteRow:
          case DataPreprocessingChangedEventType.Any:
          case DataPreprocessingChangedEventType.Transformation:
            int maxRowIndex = Math.Max(0, Rows);
            TrainingPartition.Start = Math.Min(TrainingPartition.Start, maxRowIndex);
            TrainingPartition.End = Math.Min(TrainingPartition.End, maxRowIndex);
            TestPartition.Start = Math.Min(TestPartition.Start, maxRowIndex);
            TestPartition.End = Math.Min(TestPartition.End, maxRowIndex);
            break;
        }
      };
    }
    #endregion

    #region Cells
    public bool IsCellEmpty(int columnIndex, int rowIndex) {
      return !dataColumns[columnIndex].IsValidValue(rowIndex);
    }

    private void ColumnTypeSwitchAction<T>(int columnIndex, T value, Action<DoublePreprocessingDataColumn, double?> doubleAction,
      Action<StringPreprocessingDataColumn, string> stringAction = null, Action<DateTimePreprocessingDataColumn, DateTime?> dateTimeAction = null) {
      ColumnTypeSwitchAction(dataColumns[columnIndex], value, doubleAction, stringAction, dateTimeAction);
    }
    private void ColumnTypeSwitchAction<T>(PreprocessingDataColumn column, T value, Action<DoublePreprocessingDataColumn, double?> doubleAction,
      Action<StringPreprocessingDataColumn, string> stringAction = null, Action<DateTimePreprocessingDataColumn, DateTime?> dateTimeAction = null) {
      var doubleColumn = column as DoublePreprocessingDataColumn;
      if (doubleColumn != null && doubleAction != null) doubleAction(doubleColumn, Convert<double?>(value));
      var stringColumn = column as StringPreprocessingDataColumn;
      if (stringColumn != null && stringAction != null) stringAction(stringColumn, Convert<string>(value));
      var dateTimeColumn = column as DateTimePreprocessingDataColumn;
      if (dateTimeColumn != null && dateTimeAction != null) dateTimeAction(dateTimeColumn, Convert<DateTime?>(value));
    }

    private void ColumnTypeSwitchAction(int columnIndex, Action<DoublePreprocessingDataColumn> doubleAction,
      Action<StringPreprocessingDataColumn> stringAction = null, Action<DateTimePreprocessingDataColumn> dateTimeAction = null) {
      ColumnTypeSwitchAction(dataColumns[columnIndex], doubleAction, stringAction, dateTimeAction);
    }
    private void ColumnTypeSwitchAction(PreprocessingDataColumn column, Action<DoublePreprocessingDataColumn> doubleAction,
      Action<StringPreprocessingDataColumn> stringAction = null, Action<DateTimePreprocessingDataColumn> dateTimeAction = null) {
      var doubleColumn = column as DoublePreprocessingDataColumn;
      if (doubleColumn != null && doubleAction != null) doubleAction(doubleColumn);
      var stringColumn = column as StringPreprocessingDataColumn;
      if (stringColumn != null && stringAction != null) stringAction(stringColumn);
      var dateTimeColumn = column as DateTimePreprocessingDataColumn;
      if (dateTimeColumn != null && dateTimeAction != null) dateTimeAction(dateTimeColumn);
    }


    private T ColumnTypeSwitchFunc<T>(int columnIndex, Func<DoublePreprocessingDataColumn, double?> doubleFunc,
      Func<StringPreprocessingDataColumn, string> stringFunc = null, Func<DateTimePreprocessingDataColumn, DateTime?> dateTimeFunc = null) {
      var doubleColumn = dataColumns[columnIndex] as DoublePreprocessingDataColumn;
      if (doubleColumn != null && doubleFunc != null) return Convert<T>(doubleFunc(doubleColumn));
      var stringColumn = dataColumns[columnIndex] as StringPreprocessingDataColumn;
      if (stringColumn != null && stringFunc != null) return Convert<T>(stringFunc(stringColumn));
      var dateTimeColumn = dataColumns[columnIndex] as DateTimePreprocessingDataColumn;
      if (dateTimeColumn != null && dateTimeFunc != null) return Convert<T>(dateTimeFunc(dateTimeColumn));
      throw new InvalidOperationException("Invalid data column type.");
    }

    private T ColumnTypeSwitchFuncResult<T>(int columnIndex, Func<DoublePreprocessingDataColumn, T> doubleFunc,
      Func<StringPreprocessingDataColumn, T> stringFunc = null, Func<DateTimePreprocessingDataColumn, T> dateTimeFunc = null) {
      var doubleColumn = dataColumns[columnIndex] as DoublePreprocessingDataColumn;
      if (doubleColumn != null && doubleFunc != null) return doubleFunc(doubleColumn);
      var stringColumn = dataColumns[columnIndex] as StringPreprocessingDataColumn;
      if (stringColumn != null && stringFunc != null) return stringFunc(stringColumn);
      var dateTimeColumn = dataColumns[columnIndex] as DateTimePreprocessingDataColumn;
      if (dateTimeColumn != null && dateTimeFunc != null) return dateTimeFunc(dateTimeColumn);
      throw new InvalidOperationException("Invalid data column type.");
    }
    private TOut ColumnTypeSwitchFuncResult<TIn, TOut>(int columnIndex, TIn value, Func<DoublePreprocessingDataColumn, double?, TOut> doubleFunc,
     Func<StringPreprocessingDataColumn, string, TOut> stringFunc = null, Func<DateTimePreprocessingDataColumn, DateTime?, TOut> dateTimeFunc = null) {
      var doubleColumn = dataColumns[columnIndex] as DoublePreprocessingDataColumn;
      if (doubleColumn != null && doubleFunc != null) return doubleFunc(doubleColumn, Convert<double?>(value));
      var stringColumn = dataColumns[columnIndex] as StringPreprocessingDataColumn;
      if (stringColumn != null && stringFunc != null) return stringFunc(stringColumn, Convert<string>(value));
      var dateTimeColumn = dataColumns[columnIndex] as DateTimePreprocessingDataColumn;
      if (dateTimeColumn != null && dateTimeFunc != null) return dateTimeFunc(dateTimeColumn, Convert<DateTime?>(value));
      throw new InvalidOperationException("Invalid data column type.");
    }

    private IList<T> ColumnTypeSwitchFuncList<T>(int columnIndex, Func<DoublePreprocessingDataColumn, IList<double>> doubleFunc,
      Func<StringPreprocessingDataColumn, IList<string>> stringFunc = null, Func<DateTimePreprocessingDataColumn, IList<DateTime>> dateTimeFunc = null) {
      var doubleColumn = dataColumns[columnIndex] as DoublePreprocessingDataColumn;
      if (doubleColumn != null && doubleFunc != null) return Convert<IList<T>>(doubleFunc(doubleColumn));
      var stringColumn = dataColumns[columnIndex] as StringPreprocessingDataColumn;
      if (stringColumn != null && stringFunc != null) return Convert<IList<T>>(stringFunc(stringColumn));
      var dateTimeColumn = dataColumns[columnIndex] as DateTimePreprocessingDataColumn;
      if (dateTimeColumn != null && dateTimeFunc != null) return Convert<IList<T>>(dateTimeFunc(dateTimeColumn));
      throw new InvalidOperationException("Invalid data column type.");
    }
    private static T Convert<T>(object obj) { return (T)obj; }


    public T GetCell<T>(int columnIndex, int rowIndex) {
      return ColumnTypeSwitchFunc<T>(columnIndex,
        c => c[rowIndex],
        c => c[rowIndex],
        c => c[rowIndex]);
    }

    public void SetCell<T>(int columnIndex, int rowIndex, T value) {
      SaveSnapshot(DataPreprocessingChangedEventType.ChangeItem, columnIndex, rowIndex);

      for (int i = Rows; i <= rowIndex; i++)
        InsertRow(i);
      for (int i = Columns; i <= columnIndex; i++)
        InsertColumn<T>(i.ToString(), i);

      ColumnTypeSwitchAction<T>(columnIndex, value,
        (c, v) => c[rowIndex] = v,
        (c, v) => c[rowIndex] = v,
        (c, v) => c[rowIndex] = v);

      if (!IsInTransaction)
        OnChanged(DataPreprocessingChangedEventType.ChangeItem, columnIndex, rowIndex);
    }

    public string GetCellAsString(int columnIndex, int rowIndex) {
      return dataColumns[columnIndex].GetValue(rowIndex);
    }

    public IList<T> GetValues<T>(int columnIndex, bool considerSelection) {
      if (considerSelection) {
        var list = new List<T>();
        foreach (var rowIdx in selection[columnIndex]) {
          list.Add(GetCell<T>(columnIndex, rowIdx));
          //list.Add((T)dataColumns[columnIndex][rowIdx]);
        }
        return list;
      } else {
        return ColumnTypeSwitchFuncList<T>(columnIndex,
          c => c.Values.Select(x => x ?? double.NaN).ToList(),
          c => c.Values,
          c => c.Values.Select(x => x ?? DateTime.MinValue).ToList());
        //(IList<T>)dataColumns[columnIndex];
      }
    }

    public void SetValues<T>(int columnIndex, IList<T> values) {
      SaveSnapshot(DataPreprocessingChangedEventType.ChangeColumn, columnIndex, -1);
      if (VariableHasType<T>(columnIndex)) {
        var name = dataColumns[columnIndex].Name;
        if (dataColumns[columnIndex].IsType<double>()) {
          dataColumns[columnIndex] = new DoublePreprocessingDataColumn(name, (IEnumerable<double>)values);
        } else if (dataColumns[columnIndex].IsType<string>()) {
          dataColumns[columnIndex] = new StringPreprocessingDataColumn(name, (IEnumerable<string>)values);
        } else if (dataColumns[columnIndex].IsType<DateTime>()) {
          dataColumns[columnIndex] = new DateTimePreprocessingDataColumn(name, (IEnumerable<DateTime>)values);
        } else {
          throw new ArgumentException("Unknown column type");
        }
      } else {
        throw new ArgumentException("The datatype of column " + columnIndex + " must be of type " + dataColumns[columnIndex].GetType().Name + " but was " + typeof(T).Name);
      }
      if (!IsInTransaction)
        OnChanged(DataPreprocessingChangedEventType.ChangeColumn, columnIndex, -1);
    }

    public bool SetValue(string value, int columnIndex, int rowIndex) {
      bool valid = false;
      if (VariableHasType<double>(columnIndex)) {
        double val;
        if (string.IsNullOrWhiteSpace(value)) {
          val = double.NaN;
          valid = true;
        } else {
          valid = double.TryParse(value, out val);
        }
        if (valid)
          SetCell(columnIndex, rowIndex, val);
      } else if (VariableHasType<string>(columnIndex)) {
        valid = value != null;
        if (valid)
          SetCell(columnIndex, rowIndex, value);
      } else if (VariableHasType<DateTime>(columnIndex)) {
        DateTime date;
        valid = DateTime.TryParse(value, out date);
        if (valid)
          SetCell(columnIndex, rowIndex, date);
      } else {
        throw new ArgumentException("column " + columnIndex + " contains a non supported type.");
      }

      if (!IsInTransaction)
        OnChanged(DataPreprocessingChangedEventType.ChangeColumn, columnIndex, -1);

      return valid;
    }

    public int Columns {
      get { return dataColumns.Count; }
    }

    public int Rows {
      get { return dataColumns.Count > 0 ? dataColumns[0].Length : 0; }
    }
    #endregion

    #region Rows
    public void InsertRow(int rowIndex) {
      SaveSnapshot(DataPreprocessingChangedEventType.DeleteRow, -1, rowIndex);
      foreach (var column in dataColumns) {
        ColumnTypeSwitchAction(column,
          c => c.Values.Insert(rowIndex, null),
          c => c.Values.Insert(rowIndex, null),
          c => c.Values.Insert(rowIndex, null));
        //var valueType = column.GetValueType();
        //column.Insert(rowIndex, valueType.IsValueType ? Activator.CreateInstance(valueType) : null);
      }
      if (TrainingPartition.Start <= rowIndex && rowIndex <= TrainingPartition.End) {
        TrainingPartition.End++;
        if (TrainingPartition.End <= TestPartition.Start) {
          TestPartition.Start++;
          TestPartition.End++;
        }
      } else if (TestPartition.Start <= rowIndex && rowIndex <= TestPartition.End) {
        TestPartition.End++;
        if (TestPartition.End <= TrainingPartition.Start) {
          TestPartition.Start++;
          TestPartition.End++;
        }
      }
      if (!IsInTransaction)
        OnChanged(DataPreprocessingChangedEventType.AddRow, -1, rowIndex);
    }
    public void DeleteRow(int rowIndex) {
      SaveSnapshot(DataPreprocessingChangedEventType.AddRow, -1, rowIndex);
      foreach (var column in dataColumns) {
        ColumnTypeSwitchAction(column,
          c => c.Values.RemoveAt(rowIndex),
          c => c.Values.RemoveAt(rowIndex),
          c => c.Values.RemoveAt(rowIndex));
        //column.RemoveAt(rowIndex);
      }
      if (TrainingPartition.Start <= rowIndex && rowIndex <= TrainingPartition.End) {
        TrainingPartition.End--;
        if (TrainingPartition.End <= TestPartition.Start) {
          TestPartition.Start--;
          TestPartition.End--;
        }
      } else if (TestPartition.Start <= rowIndex && rowIndex <= TestPartition.End) {
        TestPartition.End--;
        if (TestPartition.End <= TrainingPartition.Start) {
          TestPartition.Start--;
          TestPartition.End--;
        }
      }
      if (!IsInTransaction)
        OnChanged(DataPreprocessingChangedEventType.DeleteRow, -1, rowIndex);
    }
    public void DeleteRowsWithIndices(IEnumerable<int> rows) {
      SaveSnapshot(DataPreprocessingChangedEventType.AddRow, -1, -1);
      foreach (int rowIndex in rows.OrderByDescending(x => x)) {
        foreach (var column in dataColumns) {
          ColumnTypeSwitchAction(column,
            c => c.Values.RemoveAt(rowIndex),
            c => c.Values.RemoveAt(rowIndex),
            c => c.Values.RemoveAt(rowIndex));
          //column.RemoveAt(rowIndex);
        }
        if (TrainingPartition.Start <= rowIndex && rowIndex <= TrainingPartition.End) {
          TrainingPartition.End--;
          if (TrainingPartition.End <= TestPartition.Start) {
            TestPartition.Start--;
            TestPartition.End--;
          }
        } else if (TestPartition.Start <= rowIndex && rowIndex <= TestPartition.End) {
          TestPartition.End--;
          if (TestPartition.End <= TrainingPartition.Start) {
            TestPartition.Start--;
            TestPartition.End--;
          }
        }
      }
      if (!IsInTransaction)
        OnChanged(DataPreprocessingChangedEventType.DeleteRow, -1, -1);
    }

    public void InsertColumn<T>(string variableName, int columnIndex) {
      SaveSnapshot(DataPreprocessingChangedEventType.DeleteColumn, columnIndex, -1);

      if (typeof(T) == typeof(double)) {
        dataColumns.Insert(columnIndex, new DoublePreprocessingDataColumn(variableName, Enumerable.Repeat<double?>(null, Rows)));
      } else if (typeof(T) == typeof(string)) {
        dataColumns.Add(new StringPreprocessingDataColumn(variableName, Enumerable.Repeat<string>(null, Rows)));
      } else if (typeof(T) == typeof(DateTime)) {
        dataColumns.Add(new DateTimePreprocessingDataColumn(variableName, Enumerable.Repeat<DateTime?>(null, Rows)));
      } else {
        throw new ArgumentException("The datatype of column " + variableName + " must be of type double, string or DateTime");
      }

      //dataColumns.Insert(columnIndex, new List<T>(Enumerable.Repeat(default(T), Rows)));
      //variableNames.Insert(columnIndex, variableName);
      if (!IsInTransaction)
        OnChanged(DataPreprocessingChangedEventType.AddColumn, columnIndex, -1);
    }

    public void DeleteColumn(int columnIndex) {
      SaveSnapshot(DataPreprocessingChangedEventType.AddColumn, columnIndex, -1);
      dataColumns.RemoveAt(columnIndex);
      //variableNames.RemoveAt(columnIndex);
      if (!IsInTransaction)
        OnChanged(DataPreprocessingChangedEventType.DeleteColumn, columnIndex, -1);
    }

    public void RenameColumn(int columnIndex, string name) {
      SaveSnapshot(DataPreprocessingChangedEventType.ChangeColumn, columnIndex, -1);
      if (columnIndex < 0 || columnIndex > dataColumns.Count)
        throw new ArgumentOutOfRangeException("columnIndex");
      dataColumns[columnIndex].Name = name;

      if (!IsInTransaction)
        OnChanged(DataPreprocessingChangedEventType.ChangeColumn, -1, -1);
    }

    public void RenameColumns(IList<string> names) {
      if (names == null) throw new ArgumentNullException("names");
      if (names.Count != dataColumns.Count) throw new ArgumentException("number of names must match the number of columns.", "names");

      SaveSnapshot(DataPreprocessingChangedEventType.ChangeColumn, -1, -1);
      for (int i = 0; i < names.Count; i++)
        dataColumns[i].Name = names[i];

      if (!IsInTransaction)
        OnChanged(DataPreprocessingChangedEventType.ChangeColumn, -1, -1);
    }

    public bool AreAllStringColumns(IEnumerable<int> columnIndices) {
      return columnIndices.All(x => VariableHasType<string>(x));
    }
    #endregion

    #region Variables
    public IEnumerable<string> VariableNames {
      get { return dataColumns.Select(c => c.Name); }
    }

    public IEnumerable<string> GetDoubleVariableNames() {
      return dataColumns.OfType<DoublePreprocessingDataColumn>().Select(c => c.Name);
    }

    public string GetVariableName(int columnIndex) {
      return dataColumns[columnIndex].Name;
    }

    public int GetColumnIndex(string variableName) {
      return dataColumns.FindIndex(c => c.Name == variableName);
    }

    public bool VariableHasType<T>(int columnIndex) {
      return dataColumns[columnIndex].IsType<T>();
    }

    public Type GetVariableType(int columnIndex) {
      return dataColumns[columnIndex].GetValueType();
    }

    public IList<string> InputVariables { get; private set; }
    public string TargetVariable { get; private set; } // optional
    #endregion

    #region Partitions
    [Storable]
    public IntRange TrainingPartition { get; set; }
    [Storable]
    public IntRange TestPartition { get; set; }
    #endregion

    #region Transformations
    [Storable]
    public IList<ITransformation> Transformations { get; protected set; }
    #endregion

    #region Validation
    public bool Validate(string value, out string errorMessage, int columnIndex) {
      if (columnIndex < 0 || columnIndex > VariableNames.Count()) {
        throw new ArgumentOutOfRangeException("column index is out of range");
      }

      bool valid = false;
      errorMessage = string.Empty;
      if (VariableHasType<double>(columnIndex)) {
        if (string.IsNullOrWhiteSpace(value)) {
          valid = true;
        } else {
          double val;
          valid = double.TryParse(value, out val);
          if (!valid) {
            errorMessage = "Invalid Value (Valid Value Format: \"" + FormatPatterns.GetDoubleFormatPattern() + "\")";
          }
        }
      } else if (VariableHasType<string>(columnIndex)) {
        valid = value != null;
        if (!valid) {
          errorMessage = "Invalid Value (string must not be null)";
        }
      } else if (VariableHasType<DateTime>(columnIndex)) {
        DateTime date;
        valid = DateTime.TryParse(value, out date);
        if (!valid) {
          errorMessage = "Invalid Value (Valid Value Format: \"" + CultureInfo.CurrentCulture.DateTimeFormat + "\"";
        }
      } else {
        throw new ArgumentException("column " + columnIndex + " contains a non supported type.");
      }

      return valid;
    }
    #endregion

    #region Import & Export
    public void Import(IDataAnalysisProblemData problemData) {
      var dataset = problemData.Dataset;
      InputVariables = new List<string>(problemData.AllowedInputVariables);
      TargetVariable = problemData is IRegressionProblemData ? ((IRegressionProblemData)problemData).TargetVariable
        : problemData is IClassificationProblemData ? ((IClassificationProblemData)problemData).TargetVariable
        : null;

      dataColumns.Clear();
      foreach (var variableName in problemData.Dataset.VariableNames) {
        if (dataset.VariableHasType<double>(variableName)) {
          dataColumns.Add(new DoublePreprocessingDataColumn(variableName, dataset.GetDoubleValues(variableName)));
        } else if (dataset.VariableHasType<string>(variableName)) {
          dataColumns.Add(new StringPreprocessingDataColumn(variableName, dataset.GetStringValues(variableName)));
        } else if (dataset.VariableHasType<DateTime>(variableName)) {
          dataColumns.Add(new DateTimePreprocessingDataColumn(variableName, dataset.GetDateTimeValues(variableName)));
        } else {
          throw new ArgumentException("The datatype of column " + variableName + " must be of type double, string or DateTime");
        }
      }

      TrainingPartition = new IntRange(problemData.TrainingPartition.Start, problemData.TrainingPartition.End);
      TestPartition = new IntRange(problemData.TestPartition.Start, problemData.TestPartition.End);
    }

    public Dataset ExportToDataset() {
      IList<IList> values = new List<IList>();

      for (int i = 0; i < Columns; i++) {
        var doubleColumn = dataColumns[i] as DoublePreprocessingDataColumn;
        var stringColumn = dataColumns[i] as StringPreprocessingDataColumn;
        var dateTimeColumn = dataColumns[i] as DateTimePreprocessingDataColumn;
        if (doubleColumn != null) values.Add(new List<double>(doubleColumn.Values.Select(x => x ?? double.NaN)));
        else if (stringColumn != null) values.Add(new List<string>(stringColumn.Values));
        else if (dateTimeColumn != null) values.Add(new List<DateTime>(dateTimeColumn.Values.Select(x => x ?? DateTime.MinValue)));
        else throw new InvalidOperationException("Column type not supported for export");
      }

      return new Dataset(VariableNames, values);
    }
    #endregion

    #region Selection
    [Storable]
    protected IDictionary<int, IList<int>> selection;
    public IDictionary<int, IList<int>> Selection {
      get { return selection; }
      set {
        selection = value;
        OnSelectionChanged();
      }
    }
    public void ClearSelection() {
      Selection = new Dictionary<int, IList<int>>();
    }

    public event EventHandler SelectionChanged;
    protected void OnSelectionChanged() {
      var listeners = SelectionChanged;
      if (listeners != null) listeners(this, EventArgs.Empty);
    }
    #endregion

    #region Transactions
    // Snapshot/History are not storable/cloneable on purpose
    private class Snapshot {
      public List<PreprocessingDataColumn> DataColumns { get; set; }

      public IntRange TrainingPartition { get; set; }
      public IntRange TestPartition { get; set; }
      public IList<ITransformation> Transformations { get; set; }
      public DataPreprocessingChangedEventType ChangedType { get; set; }

      public int ChangedColumn { get; set; }
      public int ChangedRow { get; set; }
    }

    public event DataPreprocessingChangedEventHandler Changed;
    protected virtual void OnChanged(DataPreprocessingChangedEventType type, int column, int row) {
      var listeners = Changed;
      if (listeners != null) listeners(this, new DataPreprocessingChangedEventArgs(type, column, row));
    }

    private const int MaxUndoDepth = 5;

    private readonly IList<Snapshot> undoHistory = new List<Snapshot>();
    private readonly Stack<DataPreprocessingChangedEventType> eventStack = new Stack<DataPreprocessingChangedEventType>();

    public bool IsInTransaction { get { return eventStack.Count > 0; } }

    private void SaveSnapshot(DataPreprocessingChangedEventType changedType, int column, int row) {
      if (IsInTransaction) return;

      var cloner = new Cloner();
      var currentSnapshot = new Snapshot {
        DataColumns = new List<PreprocessingDataColumn>(dataColumns.Select(cloner.Clone)),
        TrainingPartition = new IntRange(TrainingPartition.Start, TrainingPartition.End),
        TestPartition = new IntRange(TestPartition.Start, TestPartition.End),
        Transformations = new List<ITransformation>(Transformations),
        ChangedType = changedType,
        ChangedColumn = column,
        ChangedRow = row
      };

      if (undoHistory.Count >= MaxUndoDepth)
        undoHistory.RemoveAt(0);

      undoHistory.Add(currentSnapshot);
    }

    public bool IsUndoAvailable {
      get { return undoHistory.Count > 0; }
    }

    public void Undo() {
      if (IsUndoAvailable) {
        Snapshot previousSnapshot = undoHistory[undoHistory.Count - 1];
        dataColumns = previousSnapshot.DataColumns;
        TrainingPartition = previousSnapshot.TrainingPartition;
        TestPartition = previousSnapshot.TestPartition;
        Transformations = previousSnapshot.Transformations;
        undoHistory.Remove(previousSnapshot);
        OnChanged(previousSnapshot.ChangedType,
          previousSnapshot.ChangedColumn,
          previousSnapshot.ChangedRow);
      }
    }

    public void InTransaction(Action action, DataPreprocessingChangedEventType type = DataPreprocessingChangedEventType.Any) {
      BeginTransaction(type);
      action();
      EndTransaction();
    }

    public void BeginTransaction(DataPreprocessingChangedEventType type) {
      SaveSnapshot(type, -1, -1);
      eventStack.Push(type);
    }

    public void EndTransaction() {
      if (eventStack.Count == 0)
        throw new InvalidOperationException("There is no open transaction that can be ended.");

      var @event = eventStack.Pop();
      OnChanged(@event, -1, -1);
    }
    #endregion

    #region Statistics
    public T GetMin<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
      return values.Any() ? values.Min() : emptyValue;
    }

    public T GetMax<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
      return values.Any() ? values.Max() : emptyValue;
    }

    public T GetMean<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
      if (typeof(T) == typeof(double)) {
        var values = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
        return values.Any() ? Convert<T>(values.Average()) : emptyValue;
      }
      if (typeof(T) == typeof(string)) {
        return Convert<T>(string.Empty);
      }
      if (typeof(T) == typeof(DateTime)) {
        var values = GetValuesWithoutMissingValues<DateTime>(columnIndex, considerSelection);
        return values.Any() ? Convert<T>(AggregateAsDouble(values, Enumerable.Average)) : emptyValue;
      }

      throw new InvalidOperationException(typeof(T) + " not supported");
    }

    public T GetMedian<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IComparable<T> {
      if (typeof(T) == typeof(double)) {// IEnumerable<double> is faster  
        var doubleValues = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
        return doubleValues.Any() ? Convert<T>(doubleValues.Median()) : emptyValue;
      }
      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
      return values.Any() ? values.Quantile(0.5) : emptyValue;
    }

    public T GetMode<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IEquatable<T> {
      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
      return values.Any() ? values.GroupBy(x => x).OrderByDescending(g => g.Count()).Select(g => g.Key).First() : emptyValue;
    }

    public T GetStandardDeviation<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
      if (typeof(T) == typeof(double)) {
        var values = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
        return values.Any() ? Convert<T>(values.StandardDeviation()) : emptyValue;
      }
      // For DateTime, std.dev / variance would have to be TimeSpan
      //if (typeof(T) == typeof(DateTime)) {
      //  var values = GetValuesWithoutMissingValues<DateTime>(columnIndex, considerSelection);
      //  return values.Any() ? Convert<T>(AggregateAsDouble(values, EnumerableStatisticExtensions.StandardDeviation)) : emptyValue;
      //}
      return default(T);
    }

    public T GetVariance<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
      if (typeof(T) == typeof(double)) {
        var values = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
        return values.Any() ? Convert<T>(values.Variance()) : emptyValue;
      }
      // DateTime variance often overflows long, thus the corresponding DateTime is invalid
      //if (typeof(T) == typeof(DateTime)) {
      //  var values = GetValuesWithoutMissingValues<DateTime>(columnIndex, considerSelection);
      //  return values.Any() ? Convert<T>(AggregateAsDouble(values, EnumerableStatisticExtensions.Variance)) : emptyValue;
      //}
      return default(T);
    }

    public T GetQuantile<T>(double alpha, int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IComparable<T> {
      if (typeof(T) == typeof(double)) {// IEnumerable<double> is faster  
        var doubleValues = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
        return doubleValues.Any() ? Convert<T>(doubleValues.Quantile(alpha)) : emptyValue;
      }
      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
      return values.Any() ? values.Quantile(alpha) : emptyValue;
    }

    public int GetDistinctValues<T>(int columnIndex, bool considerSelection = false) {
      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
      return values.GroupBy(x => x).Count();
    }

    private IEnumerable<T> GetValuesWithoutMissingValues<T>(int columnIndex, bool considerSelection) {
      //var doubleColumn = dataColumns[columnIndex] as DoublePreprocessingDataColumn;
      //var stringColumn = dataColumns[columnIndex] as StringPreprocessingDataColumn;
      //var dateTimeColumn = dataColumns[columnIndex] as DateTimePreprocessingDataColumn;
      //return GetValues<T>(columnIndex, considerSelection).Where(x =>
      //  doubleColumn != null ? doubleColumn.IsValidValue(Convert<double>(x))
      //  : stringColumn != null ? stringColumn.IsValidValue(Convert<string>(x))
      //  : dateTimeColumn != null ? dateTimeColumn.IsValidValue(Convert<DateTime>(x))
      //  : false);
      //!IsMissingValue(x));

      return GetValues<T>(columnIndex, considerSelection).Where(x =>
        ColumnTypeSwitchFuncResult<T, bool>(columnIndex, x,
          (c, v) => v.HasValue && c.IsValidValue(v.Value),
          (c, v) => c.IsValidValue(v),
          (c, v) => v.HasValue && c.IsValidValue(v.Value)
      ));
    }

    private static DateTime AggregateAsDouble(IEnumerable<DateTime> values, Func<IEnumerable<double>, double> func) {
      return new DateTime((long)(func(values.Select(x => (double)x.Ticks / TimeSpan.TicksPerSecond)) * TimeSpan.TicksPerSecond));
    }

    public int GetMissingValueCount() {
      int count = 0;
      for (int i = 0; i < Columns; ++i) {
        count += GetMissingValueCount(i);
      }
      return count;
    }
    public int GetMissingValueCount(int columnIndex) {
      int sum = 0;
      for (int i = 0; i < Rows; i++) {
        if (IsCellEmpty(columnIndex, i))
          sum++;
      }
      return sum;
    }
    public int GetRowMissingValueCount(int rowIndex) {
      int sum = 0;
      for (int i = 0; i < Columns; i++) {
        if (IsCellEmpty(i, rowIndex))
          sum++;
      }
      return sum;
    }
    #endregion

    #region Helpers
    private static IList<IList> CopyVariableValues(IList<IList> original) {
      var copy = new List<IList>(original);
      for (int i = 0; i < original.Count; ++i) {
        copy[i] = (IList)Activator.CreateInstance(original[i].GetType(), original[i]);
      }
      return copy;
    }
    #endregion
  }

  // Adapted from HeuristicLab.Common.EnumerableStatisticExtensions
  internal static class EnumerableExtensions {
    public static T Quantile<T>(this IEnumerable<T> values, double alpha) where T : IComparable<T> {
      T[] valuesArr = values.ToArray();
      int n = valuesArr.Length;
      if (n == 0) throw new InvalidOperationException("Enumeration contains no elements.");

      var pos = n * alpha;

      return Select((int)Math.Ceiling(pos) - 1, valuesArr);

    }

    private static T Select<T>(int k, T[] arr) where T : IComparable<T> {
      int i, ir, j, l, mid, n = arr.Length;
      T a;
      l = 0;
      ir = n - 1;
      for (;;) {
        if (ir <= l + 1) {
          // Active partition contains 1 or 2 elements.
          if (ir == l + 1 && arr[ir].CompareTo(arr[l]) < 0) {
            // Case of 2 elements.
            Swap(arr, l, ir);
          }
          return arr[k];
        } else {
          mid = (l + ir) >> 1; // Choose median of left, center, and right elements
          Swap(arr, mid, l + 1); // as partitioning element a. Also

          if (arr[l].CompareTo(arr[ir]) > 0) {  // rearrange so that arr[l] arr[ir] <= arr[l+1],
            Swap(arr, l, ir); // . arr[ir] >= arr[l+1]
          }

          if (arr[l + 1].CompareTo(arr[ir]) > 0) {
            Swap(arr, l + 1, ir);
          }
          if (arr[l].CompareTo(arr[l + 1]) > 0) {
            Swap(arr, l, l + 1);
          }
          i = l + 1; // Initialize pointers for partitioning.
          j = ir;
          a = arr[l + 1]; // Partitioning element.
          for (;;) { // Beginning of innermost loop.
            do i++; while (arr[i].CompareTo(a) < 0); // Scan up to find element > a.
            do j--; while (arr[j].CompareTo(a) > 0); // Scan down to find element < a.
            if (j < i) break; // Pointers crossed. Partitioning complete.
            Swap(arr, i, j);
          } // End of innermost loop.
          arr[l + 1] = arr[j]; // Insert partitioning element.
          arr[j] = a;
          if (j >= k) ir = j - 1; // Keep active the partition that contains the
          if (j <= k) l = i; // kth element.
        }
      }
    }

    private static void Swap<T>(T[] arr, int i, int j) {
      T temp = arr[i];
      arr[i] = arr[j];
      arr[j] = temp;
    }
  }
}